summaryrefslogtreecommitdiffstats
path: root/src/main/java/org/onap
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/org/onap')
-rw-r--r--src/main/java/org/onap/aai/GraphAdminApp.java129
-rw-r--r--src/main/java/org/onap/aai/Profiles.java31
-rw-r--r--src/main/java/org/onap/aai/config/AuditorConfiguration.java34
-rw-r--r--src/main/java/org/onap/aai/config/DslConfiguration.java44
-rw-r--r--src/main/java/org/onap/aai/config/JettyPasswordDecoder.java33
-rw-r--r--src/main/java/org/onap/aai/config/PasswordDecoder.java25
-rw-r--r--src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java78
-rw-r--r--src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java317
-rw-r--r--src/main/java/org/onap/aai/datagrooming/DataGrooming.java2853
-rw-r--r--src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java204
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java835
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java115
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java421
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java223
-rw-r--r--src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java107
-rw-r--r--src/main/java/org/onap/aai/db/schema/AuditDoc.java88
-rw-r--r--src/main/java/org/onap/aai/db/schema/AuditJanusGraph.java121
-rw-r--r--src/main/java/org/onap/aai/db/schema/AuditOXM.java227
-rw-r--r--src/main/java/org/onap/aai/db/schema/Auditor.java53
-rw-r--r--src/main/java/org/onap/aai/db/schema/AuditorFactory.java53
-rw-r--r--src/main/java/org/onap/aai/db/schema/CompareByName.java35
-rw-r--r--src/main/java/org/onap/aai/db/schema/DBIndex.java104
-rw-r--r--src/main/java/org/onap/aai/db/schema/DBProperty.java83
-rw-r--r--src/main/java/org/onap/aai/db/schema/EdgeProperty.java68
-rw-r--r--src/main/java/org/onap/aai/db/schema/ManageJanusGraphSchema.java328
-rw-r--r--src/main/java/org/onap/aai/db/schema/Named.java30
-rw-r--r--src/main/java/org/onap/aai/db/schema/ScriptDriver.java123
-rw-r--r--src/main/java/org/onap/aai/dbgen/DupeTool.java1854
-rw-r--r--src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java875
-rw-r--r--src/main/java/org/onap/aai/dbgen/GraphMLTokens.java56
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java177
-rw-r--r--src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java317
-rw-r--r--src/main/java/org/onap/aai/dbgen/tags/Command.java25
-rw-r--r--src/main/java/org/onap/aai/interceptors/AAIContainerFilter.java41
-rw-r--r--src/main/java/org/onap/aai/interceptors/AAIHeaderProperties.java39
-rw-r--r--src/main/java/org/onap/aai/interceptors/package-info.java36
-rw-r--r--src/main/java/org/onap/aai/interceptors/post/AAIResponseFilterPriority.java40
-rw-r--r--src/main/java/org/onap/aai/interceptors/post/InvalidResponseStatus.java65
-rw-r--r--src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java98
-rw-r--r--src/main/java/org/onap/aai/interceptors/post/ResponseHeaderManipulation.java64
-rw-r--r--src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java123
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/AAIRequestFilterPriority.java46
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java91
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/HttpHeaderInterceptor.java55
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/OneWaySslAuthorization.java81
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/RequestHeaderManipulation.java62
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/RequestModification.java77
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java136
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/RetiredInterceptor.java150
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java75
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java187
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java101
-rw-r--r--src/main/java/org/onap/aai/interceptors/pre/VersionLatestInterceptor.java56
-rw-r--r--src/main/java/org/onap/aai/migration/EdgeMigrator.java145
-rw-r--r--src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java288
-rw-r--r--src/main/java/org/onap/aai/migration/Enabled.java35
-rw-r--r--src/main/java/org/onap/aai/migration/EventAction.java29
-rw-r--r--src/main/java/org/onap/aai/migration/MigrationController.java78
-rw-r--r--src/main/java/org/onap/aai/migration/MigrationControllerInternal.java498
-rw-r--r--src/main/java/org/onap/aai/migration/MigrationDangerRating.java41
-rw-r--r--src/main/java/org/onap/aai/migration/MigrationPriority.java41
-rw-r--r--src/main/java/org/onap/aai/migration/Migrator.java385
-rw-r--r--src/main/java/org/onap/aai/migration/NotificationHelper.java118
-rw-r--r--src/main/java/org/onap/aai/migration/PropertyMigrator.java146
-rw-r--r--src/main/java/org/onap/aai/migration/Status.java29
-rw-r--r--src/main/java/org/onap/aai/migration/ValueMigrator.java104
-rw-r--r--src/main/java/org/onap/aai/migration/VertexMerge.java255
-rw-r--r--src/main/java/org/onap/aai/migration/v12/ContainmentDeleteOtherVPropertyMigration.java106
-rw-r--r--src/main/java/org/onap/aai/migration/v12/DeletePInterface.java131
-rw-r--r--src/main/java/org/onap/aai/migration/v12/EdgeReportForToscaMigration.java162
-rw-r--r--src/main/java/org/onap/aai/migration/v12/MigrateModelVerDistriubutionStatusProperty.java85
-rw-r--r--src/main/java/org/onap/aai/migration/v12/MigrateServiceInstanceToConfiguration.java193
-rw-r--r--src/main/java/org/onap/aai/migration/v12/SDWANSpeedChangeMigration.java258
-rw-r--r--src/main/java/org/onap/aai/migration/v12/UpdateAaiUriIndexMigration.java328
-rw-r--r--src/main/java/org/onap/aai/migration/v12/UriMigration.java180
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateBooleanDefaultsToFalse.java114
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateInMaintDefaultToFalse.java98
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantId.java85
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionId.java84
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupSubType.java64
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupType.java64
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateModelVer.java229
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigratePserverAndPnfEquipType.java157
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantId.java84
-rw-r--r--src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionId.java83
-rw-r--r--src/main/java/org/onap/aai/rest/ExceptionHandler.java127
-rw-r--r--src/main/java/org/onap/aai/rest/QueryConsumer.java217
-rw-r--r--src/main/java/org/onap/aai/rest/dsl/DslListener.java314
-rw-r--r--src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java85
-rw-r--r--src/main/java/org/onap/aai/rest/search/GenericQueryProcessor.java226
-rw-r--r--src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java45
-rw-r--r--src/main/java/org/onap/aai/rest/search/QueryProcessorType.java26
-rw-r--r--src/main/java/org/onap/aai/rest/util/EchoResponse.java122
-rw-r--r--src/main/java/org/onap/aai/schema/GenTester.java162
-rw-r--r--src/main/java/org/onap/aai/service/AuthorizationService.java109
-rw-r--r--src/main/java/org/onap/aai/service/RetiredService.java67
-rw-r--r--src/main/java/org/onap/aai/util/PositiveNumValidator.java35
-rw-r--r--src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java183
-rw-r--r--src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java105
-rw-r--r--src/main/java/org/onap/aai/util/SendMigrationNotifications.java189
-rw-r--r--src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java105
-rw-r--r--src/main/java/org/onap/aai/util/UniquePropertyCheck.java288
-rw-r--r--src/main/java/org/onap/aai/web/JerseyConfiguration.java137
-rw-r--r--src/main/java/org/onap/aai/web/LocalHostAccessLog.java67
104 files changed, 18995 insertions, 0 deletions
diff --git a/src/main/java/org/onap/aai/GraphAdminApp.java b/src/main/java/org/onap/aai/GraphAdminApp.java
new file mode 100644
index 0000000..aa9c457
--- /dev/null
+++ b/src/main/java/org/onap/aai/GraphAdminApp.java
@@ -0,0 +1,129 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.onap.aai.config.PropertyPasswordConfiguration;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.nodes.NodeIngestor;
+import org.onap.aai.util.AAIConfig;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
+import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
+import org.springframework.context.annotation.ComponentScan;
+import org.springframework.core.env.Environment;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.scheduling.annotation.EnableScheduling;
+
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.util.UUID;
+
+@SpringBootApplication
+// Scan the specific packages that has the beans/components
+// This will add the ScheduledTask that was created in aai-common
+// Add more packages where you would need to scan for files
+@ComponentScan(basePackages = {
+ "org.onap.aai.tasks",
+ "org.onap.aai.config",
+ "org.onap.aai.service",
+ "org.onap.aai.setup",
+ "org.onap.aai.rest",
+ "org.onap.aai.web",
+ "org.onap.aai.interceptors",
+ "org.onap.aai.datasnapshot",
+ "org.onap.aai.datagrooming",
+ "org.onap.aai.datacleanup"
+})
+@EnableAsync
+@EnableScheduling
+@EnableAutoConfiguration(exclude = {DataSourceAutoConfiguration.class, HibernateJpaAutoConfiguration.class})
+public class GraphAdminApp {
+
+ public static final String APP_NAME = "GraphAdmin";
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(GraphAdminApp.class);
+
+ @Autowired
+ private Environment env;
+
+ @Autowired
+ private NodeIngestor nodeIngestor;
+
+ @PostConstruct
+ private void initialize(){
+ loadDefaultProps();
+
+ LoggingContext.save();
+ LoggingContext.component("init");
+ LoggingContext.partnerName("NA");
+ LoggingContext.targetEntity(APP_NAME);
+ LoggingContext.requestId(UUID.randomUUID().toString());
+ LoggingContext.serviceName(APP_NAME);
+ LoggingContext.targetServiceName("contextInitialized");
+ }
+
+ @PreDestroy
+ public void cleanup(){
+ AAIGraph.getInstance().graphShutdown();
+ }
+
+ public static void main(String[] args) throws Exception {
+
+ loadDefaultProps();
+ SpringApplication app = new SpringApplication(GraphAdminApp.class);
+ app.setRegisterShutdownHook(true);
+ app.addInitializers(new PropertyPasswordConfiguration());
+ Environment env = app.run(args).getEnvironment();
+
+ LOGGER.info(
+ "Application '{}' is running on {}!" ,
+ env.getProperty("spring.application.name"),
+ env.getProperty("server.port")
+ );
+ // The main reason this was moved from the constructor is due
+ // to the SchemaGenerator needs the bean and during the constructor
+ // the Spring Context is not yet initialized
+
+ AAIConfig.init();
+ AAIGraph.getInstance();
+
+ System.setProperty("org.onap.aai.graphadmin.started", "true");
+ LOGGER.info("GraphAdmin MicroService Started");
+ LOGGER.error("GraphAdmin MicroService Started");
+ LOGGER.debug("GraphAdmin MicroService Started");
+ System.out.println("GraphAdmin Microservice Started");
+ }
+
+ public static void loadDefaultProps(){
+
+ if(System.getProperty("AJSC_HOME") == null){
+ System.setProperty("AJSC_HOME", ".");
+ }
+
+ if(System.getProperty("BUNDLECONFIG_DIR") == null){
+ System.setProperty("BUNDLECONFIG_DIR", "src/main/resources");
+ }
+ }
+}
diff --git a/src/main/java/org/onap/aai/Profiles.java b/src/main/java/org/onap/aai/Profiles.java
new file mode 100644
index 0000000..f0419d8
--- /dev/null
+++ b/src/main/java/org/onap/aai/Profiles.java
@@ -0,0 +1,31 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai;
+
+public final class Profiles {
+
+ public static final String DMAAP = "dmaap";
+ public static final String DME2 = "dme2";
+
+ public static final String ONE_WAY_SSL = "one-way-ssl";
+ public static final String TWO_WAY_SSL = "two-way-ssl";
+
+ private Profiles(){}
+}
diff --git a/src/main/java/org/onap/aai/config/AuditorConfiguration.java b/src/main/java/org/onap/aai/config/AuditorConfiguration.java
new file mode 100644
index 0000000..9377393
--- /dev/null
+++ b/src/main/java/org/onap/aai/config/AuditorConfiguration.java
@@ -0,0 +1,34 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.config;
+
+import org.onap.aai.db.schema.AuditorFactory;
+import org.onap.aai.introspection.LoaderFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+@Configuration
+public class AuditorConfiguration {
+
+ @Bean
+ public AuditorFactory auditorFactory(LoaderFactory loaderFactory){
+ return new AuditorFactory(loaderFactory);
+ }
+}
diff --git a/src/main/java/org/onap/aai/config/DslConfiguration.java b/src/main/java/org/onap/aai/config/DslConfiguration.java
new file mode 100644
index 0000000..74bc046
--- /dev/null
+++ b/src/main/java/org/onap/aai/config/DslConfiguration.java
@@ -0,0 +1,44 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.config;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.rest.dsl.DslListener;
+import org.onap.aai.rest.dsl.DslQueryProcessor;
+import org.springframework.beans.factory.config.ConfigurableBeanFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Scope;
+
+@Configuration
+public class DslConfiguration {
+
+ @Bean
+ @Scope(scopeName = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+ public DslListener dslListener(EdgeIngestor edgeIngestor){
+ return new DslListener(edgeIngestor);
+ }
+
+ @Bean
+ @Scope(scopeName = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
+ public DslQueryProcessor dslQueryProcessor(DslListener dslListener){
+ return new DslQueryProcessor(dslListener);
+ }
+}
diff --git a/src/main/java/org/onap/aai/config/JettyPasswordDecoder.java b/src/main/java/org/onap/aai/config/JettyPasswordDecoder.java
new file mode 100644
index 0000000..944f951
--- /dev/null
+++ b/src/main/java/org/onap/aai/config/JettyPasswordDecoder.java
@@ -0,0 +1,33 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.config;
+
+import org.eclipse.jetty.util.security.Password;
+
+public class JettyPasswordDecoder implements PasswordDecoder {
+
+ @Override
+ public String decode(String input) {
+ if (input.startsWith("OBF:")) {
+ return Password.deobfuscate(input);
+ }
+ return Password.deobfuscate("OBF:" + input);
+ }
+}
diff --git a/src/main/java/org/onap/aai/config/PasswordDecoder.java b/src/main/java/org/onap/aai/config/PasswordDecoder.java
new file mode 100644
index 0000000..0dcb845
--- /dev/null
+++ b/src/main/java/org/onap/aai/config/PasswordDecoder.java
@@ -0,0 +1,25 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.config;
+
+public interface PasswordDecoder {
+
+ String decode(String input);
+}
diff --git a/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java b/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java
new file mode 100644
index 0000000..9befb13
--- /dev/null
+++ b/src/main/java/org/onap/aai/config/PropertyPasswordConfiguration.java
@@ -0,0 +1,78 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.config;
+
+import org.springframework.context.ApplicationContextInitializer;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.core.env.ConfigurableEnvironment;
+import org.springframework.core.env.EnumerablePropertySource;
+import org.springframework.core.env.MapPropertySource;
+import org.springframework.core.env.PropertySource;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PropertyPasswordConfiguration implements ApplicationContextInitializer<ConfigurableApplicationContext> {
+
+ private static final Pattern decodePasswordPattern = Pattern.compile("password\\((.*?)\\)");
+
+ private PasswordDecoder passwordDecoder = new JettyPasswordDecoder();
+
+ @Override
+ public void initialize(ConfigurableApplicationContext applicationContext) {
+ ConfigurableEnvironment environment = applicationContext.getEnvironment();
+ for (PropertySource<?> propertySource : environment.getPropertySources()) {
+ Map<String, Object> propertyOverrides = new LinkedHashMap<>();
+ decodePasswords(propertySource, propertyOverrides);
+ if (!propertyOverrides.isEmpty()) {
+ PropertySource<?> decodedProperties = new MapPropertySource("decoded "+ propertySource.getName(), propertyOverrides);
+ environment.getPropertySources().addBefore(propertySource.getName(), decodedProperties);
+ }
+ }
+ }
+
+ private void decodePasswords(PropertySource<?> source, Map<String, Object> propertyOverrides) {
+ if (source instanceof EnumerablePropertySource) {
+ EnumerablePropertySource<?> enumerablePropertySource = (EnumerablePropertySource<?>) source;
+ for (String key : enumerablePropertySource.getPropertyNames()) {
+ Object rawValue = source.getProperty(key);
+ if (rawValue instanceof String) {
+ String decodedValue = decodePasswordsInString((String) rawValue);
+ propertyOverrides.put(key, decodedValue);
+ }
+ }
+ }
+ }
+
+ private String decodePasswordsInString(String input) {
+ if (input == null) return null;
+ StringBuffer output = new StringBuffer();
+ Matcher matcher = decodePasswordPattern.matcher(input);
+ while (matcher.find()) {
+ String replacement = passwordDecoder.decode(matcher.group(1));
+ matcher.appendReplacement(output, replacement);
+ }
+ matcher.appendTail(output);
+ return output.toString();
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java
new file mode 100644
index 0000000..a3dc708
--- /dev/null
+++ b/src/main/java/org/onap/aai/datacleanup/DataCleanupTasks.java
@@ -0,0 +1,317 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datacleanup;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.FileTime;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import org.onap.aai.exceptions.AAIException;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+
+import com.att.eelf.configuration.EELFLogger;
+
+@Component
+@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
+public class DataCleanupTasks {
+
+ private static final EELFLogger logger = EELFManager.getInstance().getLogger(DataCleanupTasks.class);
+ private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
+
+ /**The function archives/deletes files that end in .out (Ie. dataGrooming.201511111305.out) that sit in our log/data directory structure.
+ logDir is the {project_home}/logs
+ archiveDir is the ARCHIVE directory where the files will be stored after 5 days.
+ ageZip is the number of days after which the file will be moved to the ARCHIVE folder.
+ ageDelete is the number of days after which the data files will be deleted i.e after 30 days.
+ */
+ @Scheduled(cron = "${datagroomingcleanup.cron}" )
+ public void dataGroomingCleanup() throws AAIException, Exception {
+
+ logger.info("Started cron job dataGroomingCleanup @ " + simpleDateFormat.format(new Date()));
+
+ try {
+ String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs";
+ String dataGroomingDir = logDir + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataGrooming";
+ String archiveDir = dataGroomingDir + AAIConstants.AAI_FILESEP + "ARCHIVE";
+ String dataGroomingArcDir = archiveDir + AAIConstants.AAI_FILESEP + "dataGrooming";
+ File path = new File(dataGroomingDir);
+ File archivepath = new File(archiveDir);
+ File dataGroomingPath = new File(dataGroomingArcDir);
+
+ logger.info("The logDir is " + logDir);
+ logger.info("The dataGroomingDir is " + dataGroomingDir);
+ logger.info("The archiveDir is " + archiveDir );
+ logger.info("The dataGroomingArcDir is " + dataGroomingArcDir );
+
+ boolean exists = directoryExists(logDir);
+ logger.info("Directory" + logDir + "exists: " + exists);
+ if(exists == false)
+ logger.error("The directory" + logDir +"does not exists");
+
+ Integer ageZip = AAIConfig.getInt("aai.datagrooming.agezip");
+ Integer ageDelete = AAIConfig.getInt("aai.datagrooming.agedelete");
+
+ Date newAgeZip = getZipDate(ageZip);
+
+ //Iterate through the dataGroomingDir
+ File[] listFiles = path.listFiles();
+ if(listFiles != null) {
+ for(File listFile : listFiles) {
+ if (listFile.toString().contains("ARCHIVE")){
+ continue;
+ }
+ if(listFile.isFile()){
+ logger.info("The file name in dataGrooming: " +listFile.getName());
+ Date fileCreateDate = fileCreationMonthDate(listFile);
+ logger.info("The fileCreateDate in dataGrooming is " + fileCreateDate);
+ if( fileCreateDate.compareTo(newAgeZip) < 0) {
+ archive(listFile,archiveDir,dataGroomingArcDir);
+ }
+ }
+ }
+ }
+
+ Date newAgeDelete = getZipDate(ageDelete);
+ //Iterate through the archive/dataGrooming dir
+ File[] listFilesArchive = dataGroomingPath.listFiles();
+ if(listFilesArchive != null) {
+ for(File listFileArchive : listFilesArchive) {
+ if(listFileArchive.isFile()) {
+ logger.info("The file name in ARCHIVE/dataGrooming: " +listFileArchive.getName());
+ Date fileCreateDate = fileCreationMonthDate(listFileArchive);
+ logger.info("The fileCreateDate in ARCHIVE/dataGrooming is " + fileCreateDate);
+ if(fileCreateDate.compareTo(newAgeDelete) < 0) {
+ delete(listFileArchive);
+ }
+ }
+ }
+ }
+ }
+ catch (Exception e) {
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+ logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+ throw e;
+ }
+ }
+
+ /**
+ * This method checks if the directory exists
+ * @param DIR
+ *
+ */
+ public boolean directoryExists(String dir) {
+ File path = new File(dir);
+ boolean exists = path.exists();
+ return exists;
+ }
+
+ public Date getZipDate(Integer days) throws Exception {
+ return getZipDate(days, new Date());
+ }
+
+ public Date getZipDate(Integer days, Date date) throws Exception{
+
+ Calendar cal = Calendar.getInstance();
+ logger.info("The current date is " + date );
+ cal.setTime(date);
+ cal.add(Calendar.DATE, -days);
+ Date newAgeZip = cal.getTime();
+ logger.info("The newAgeDate is " +newAgeZip);
+ return newAgeZip;
+ }
+
+
+ public Date fileCreationMonthDate (File file) throws Exception {
+
+ BasicFileAttributes attr = Files.readAttributes(file.toPath(),
+ BasicFileAttributes.class);
+ FileTime time = attr.creationTime();
+ String formatted = simpleDateFormat.format( new Date( time.toMillis() ) );
+ Date d = simpleDateFormat.parse(formatted);
+ return d;
+ }
+
+ /**
+ * This method will zip the files and add it to the archive folder
+ * Checks if the archive folder exists, if not then creates one
+ * After adding the file to archive folder it deletes the file from the filepath
+ * @throws AAIException
+ * @throws Exception
+ */
+ public void archive(File file, String archiveDir, String afterArchiveDir) throws AAIException, Exception {
+
+ logger.info("Inside the archive folder");
+ String filename = file.getName();
+ logger.info("file name is " +filename);
+ File archivepath = new File(archiveDir);
+
+ String zipFile = afterArchiveDir + AAIConstants.AAI_FILESEP + filename;
+
+ File dataGroomingPath = new File(afterArchiveDir);
+
+ boolean exists = directoryExists(archiveDir);
+ logger.info("Directory" + archiveDir + "exists: " + exists);
+ if(exists == false) {
+ logger.error("The directory" + archiveDir +"does not exists so will create a new archive folder");
+ //Create an archive folder if does not exists
+ boolean flag = dataGroomingPath.mkdirs();
+ if(flag == false)
+ logger.error("Failed to create ARCHIVE folder");
+ }
+ try {
+ FileOutputStream outputstream = new FileOutputStream(zipFile + ".gz");
+ ZipOutputStream zoutputstream = new ZipOutputStream(outputstream);
+ ZipEntry ze = new ZipEntry(file.getName());
+ zoutputstream.putNextEntry(ze);
+ //read the file and write to the zipOutputStream
+ FileInputStream inputstream = new FileInputStream(file);
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len = inputstream.read(buffer)) > 0) {
+ zoutputstream.write(buffer,0,len);
+ }
+ //close all the sources
+ zoutputstream.closeEntry();
+ zoutputstream.close();
+ inputstream.close();
+ outputstream.close();
+ //Delete the file after been added to archive folder
+ delete(file);
+ logger.info("The file archived is " + file + " at " + afterArchiveDir );
+ }
+ catch (IOException e) {
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup " + e.getStackTrace());
+ logger.info("AAI_4000", "Exception running cron job for DataCleanup", e);
+ throw e;
+ }
+ }
+
+ /**
+ * This method will delete all the files from the archive folder that are older than 60 days
+ * @param file
+ */
+ public static void delete(File file) {
+
+ logger.info("Deleting the file " + file);
+ boolean deleteStatus = file.delete();
+ if(deleteStatus == false){
+ logger.error("Failed to delete the file" +file);
+ }
+ }
+
+ /**The function archives/deletes files that end in .out (Ie. dataGrooming.201511111305.out) that sit in our log/data directory structure.
+ logDir is the {project_home}/logs
+ archiveDir is the ARCHIVE directory where the files will be stored after 5 days.
+ ageZip is the number of days after which the file will be moved to the ARCHIVE folder.
+ ageDelete is the number of days after which the data files will be deleted i.e after 30 days.
+*/
+ @Scheduled(cron = "${datasnapshotcleanup.cron}" )
+ public void dataSnapshotCleanup() throws AAIException, Exception {
+
+ logger.info("Started cron job dataSnapshotCleanup @ " + simpleDateFormat.format(new Date()));
+
+ try {
+ String logDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs";
+ String dataSnapshotDir = logDir + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataSnapshots";
+ String archiveDir = dataSnapshotDir + AAIConstants.AAI_FILESEP + "ARCHIVE";
+ String dataSnapshotArcDir = archiveDir + AAIConstants.AAI_FILESEP + "dataSnapshots";
+ File path = new File(dataSnapshotDir);
+ File archivepath = new File(archiveDir);
+ File dataSnapshotPath = new File(dataSnapshotArcDir);
+
+ logger.info("The logDir is " + logDir);
+ logger.info("The dataSnapshotDir is " + dataSnapshotDir);
+ logger.info("The archiveDir is " + archiveDir );
+ logger.info("The dataSnapshotArcDir is " + dataSnapshotArcDir );
+
+ boolean exists = directoryExists(logDir);
+ logger.info("Directory" + logDir + "exists: " + exists);
+ if(exists == false)
+ logger.error("The directory" + logDir +"does not exists");
+
+ Integer ageZipSnapshot = AAIConfig.getInt("aai.datasnapshot.agezip");
+ Integer ageDeleteSnapshot = AAIConfig.getInt("aai.datasnapshot.agedelete");
+
+ Date newAgeZip = getZipDate(ageZipSnapshot);
+
+ //Iterate through the dataGroomingDir
+ File[] listFiles = path.listFiles();
+ if(listFiles != null) {
+ for(File listFile : listFiles) {
+ if (listFile.toString().contains("ARCHIVE")){
+ continue;
+ }
+ if(listFile.isFile()){
+ logger.info("The file name in dataSnapshot: " +listFile.getName());
+ Date fileCreateDate = fileCreationMonthDate(listFile);
+ logger.info("The fileCreateDate in dataSnapshot is " + fileCreateDate);
+ if( fileCreateDate.compareTo(newAgeZip) < 0) {
+ archive(listFile,archiveDir,dataSnapshotArcDir);
+ }
+ }
+ }
+ }
+
+ Date newAgeDelete = getZipDate(ageDeleteSnapshot);
+ //Iterate through the archive/dataSnapshots dir
+ File[] listFilesArchive = dataSnapshotPath.listFiles();
+ if(listFilesArchive != null) {
+ for(File listFileArchive : listFilesArchive) {
+ if(listFileArchive.isFile()) {
+ logger.info("The file name in ARCHIVE/dataSnapshot: " +listFileArchive.getName());
+ Date fileCreateDate = fileCreationMonthDate(listFileArchive);
+ logger.info("The fileCreateDate in ARCHIVE/dataSnapshot is " + fileCreateDate);
+ if(fileCreateDate.compareTo(newAgeDelete) < 0) {
+ delete(listFileArchive);
+ }
+ }
+ }
+ }
+ }
+ catch (Exception e) {
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+ logger.info("AAI_4000", "Exception running cron job for DataCleanup"+e.toString());
+ throw e;
+ }
+ }
+}
diff --git a/src/main/java/org/onap/aai/datagrooming/DataGrooming.java b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
new file mode 100644
index 0000000..6149dd9
--- /dev/null
+++ b/src/main/java/org/onap/aai/datagrooming/DataGrooming.java
@@ -0,0 +1,2853 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datagrooming;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.onap.aai.GraphAdminApp;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.edges.enums.EdgeProperty;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.*;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraph;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+
+public class DataGrooming {
+
+ private static EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataGrooming.class);
+
+ private static final String FROMAPPID = "AAI-DB";
+ private static final String TRANSID = UUID.randomUUID().toString();
+ private int dupeGrpsDeleted = 0;
+
+ private LoaderFactory loaderFactory;
+ private SchemaVersions schemaVersions;
+
+ public DataGrooming(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.schemaVersions = schemaVersions;
+ }
+
+ public void execute(String[] args){
+
+ String ver = "version"; // Placeholder
+ Boolean doAutoFix = false;
+ Boolean edgesOnlyFlag = false;
+ Boolean dontFixOrphansFlag = false;
+ Boolean skipHostCheck = false;
+ Boolean singleCommits = false;
+ Boolean dupeCheckOff = false;
+ Boolean dupeFixOn = false;
+ Boolean ghost2CheckOff = false;
+ Boolean ghost2FixOn = false;
+ Boolean neverUseCache = false;
+ Boolean skipEdgeCheckFlag = false;
+ Boolean skipIndexUpdateFix = false;
+
+ // A value of 0 means that we will not have a time-window -- we will look
+ // at all nodes of the passed-in nodeType.
+ int timeWindowMinutes = 0;
+
+ int maxRecordsToFix = AAIConstants.AAI_GROOMING_DEFAULT_MAX_FIX;
+ int sleepMinutes = AAIConstants.AAI_GROOMING_DEFAULT_SLEEP_MINUTES;
+ try {
+ String maxFixStr = AAIConfig.get("aai.grooming.default.max.fix");
+ if( maxFixStr != null && !maxFixStr.equals("") ){
+ maxRecordsToFix = Integer.parseInt(maxFixStr);
+ }
+ String sleepStr = AAIConfig.get("aai.grooming.default.sleep.minutes");
+ if( sleepStr != null && !sleepStr.equals("") ){
+ sleepMinutes = Integer.parseInt(sleepStr);
+ }
+ }
+ catch ( Exception e ){
+ // Don't worry, we'll just use the defaults that we got from AAIConstants
+ LOGGER.warn("WARNING - could not pick up aai.grooming values from aaiconfig.properties file. ");
+ }
+
+ String prevFileName = "";
+ String singleNodeType = "";
+ dupeGrpsDeleted = 0;
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dteStr = fd.getDateTime();
+
+ if (args.length > 0) {
+ // They passed some arguments in that will affect processing
+ for (int i = 0; i < args.length; i++) {
+ String thisArg = args[i];
+ if (thisArg.equals("-edgesOnly")) {
+ edgesOnlyFlag = true;
+ } else if (thisArg.equals("-autoFix")) {
+ doAutoFix = true;
+ } else if (thisArg.equals("-skipHostCheck")) {
+ skipHostCheck = true;
+ } else if (thisArg.equals("-dontFixOrphans")) {
+ dontFixOrphansFlag = true;
+ } else if (thisArg.equals("-singleCommits")) {
+ singleCommits = true;
+ } else if (thisArg.equals("-dupeCheckOff")) {
+ dupeCheckOff = true;
+ } else if (thisArg.equals("-dupeFixOn")) {
+ dupeFixOn = true;
+ } else if (thisArg.equals("-ghost2CheckOff")) {
+ ghost2CheckOff = true;
+ } else if (thisArg.equals("-neverUseCache")) {
+ neverUseCache = true;
+ } else if (thisArg.equals("-ghost2FixOn")) {
+ ghost2FixOn = true;
+ } else if (thisArg.equals("-skipEdgeChecks")) {
+ skipEdgeCheckFlag = true;
+ } else if (thisArg.equals("-skipIndexUpdateFix")) {
+ skipIndexUpdateFix = true;
+ } else if (thisArg.equals("-maxFix")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error(" No value passed with -maxFix option. ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ String nextArg = args[i];
+ try {
+ maxRecordsToFix = Integer.parseInt(nextArg);
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error("Bad value passed with -maxFix option: ["
+ + nextArg + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ } else if (thisArg.equals("-sleepMinutes")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error("No value passed with -sleepMinutes option.");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ String nextArg = args[i];
+ try {
+ sleepMinutes = Integer.parseInt(nextArg);
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error("Bad value passed with -sleepMinutes option: ["
+ + nextArg + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ } else if (thisArg.equals("-timeWindowMinutes")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error("No value passed with -timeWindowMinutes option.");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ String nextArg = args[i];
+ try {
+ timeWindowMinutes = Integer.parseInt(nextArg);
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error("Bad value passed with -timeWindowMinutes option: ["
+ + nextArg + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+
+ } else if (thisArg.equals("-f")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error(" No value passed with -f option. ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ prevFileName = args[i];
+ } else if (thisArg.equals("-singleNodeType")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error(" No value passed with -onlyThisNodeType option. ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ singleNodeType = args[i];
+ } else {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error(" Unrecognized argument passed to DataGrooming: ["
+ + thisArg + "]. ");
+ LOGGER.error(" Valid values are: -f -autoFix -maxFix -edgesOnly -skipEdgeChecks -dupeFixOn -donFixOrphans -timeWindowMinutes -sleepMinutes -neverUseCache");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+ }
+ }
+
+ String windowTag = "FULL";
+ if( timeWindowMinutes > 0 ){
+ windowTag = "PARTIAL";
+ }
+ String groomOutFileName = "dataGrooming." + windowTag + "." + dteStr + ".out";
+
+ try {
+ loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion());
+ }
+ catch (Exception ex){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error("ERROR - Could not create loader " + LogFormatTools.getStackTop(ex));
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ try {
+ if (!prevFileName.equals("")) {
+ // They are trying to fix some data based on a data in a
+ // previous file.
+ LOGGER.info(" Call doTheGrooming() with a previous fileName ["
+ + prevFileName + "] for cleanup. ");
+ Boolean finalShutdownFlag = true;
+ Boolean cacheDbOkFlag = false;
+ doTheGrooming(prevFileName, edgesOnlyFlag, dontFixOrphansFlag,
+ maxRecordsToFix, groomOutFileName, ver, singleCommits,
+ dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
+ finalShutdownFlag, cacheDbOkFlag,
+ skipEdgeCheckFlag, timeWindowMinutes,
+ singleNodeType, skipIndexUpdateFix );
+ } else if (doAutoFix) {
+ // They want us to run the processing twice -- first to look for
+ // delete candidates, then after
+ // napping for a while, run it again and delete any candidates
+ // that were found by the first run.
+ // Note: we will produce a separate output file for each of the
+ // two runs.
+ LOGGER.info(" Doing an auto-fix call to Grooming. ");
+ LOGGER.info(" First, Call doTheGrooming() to look at what's out there. ");
+ Boolean finalShutdownFlag = false;
+ Boolean cacheDbOkFlag = true;
+ int fixCandCount = doTheGrooming("", edgesOnlyFlag,
+ dontFixOrphansFlag, maxRecordsToFix, groomOutFileName,
+ ver, singleCommits, dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
+ finalShutdownFlag, cacheDbOkFlag,
+ skipEdgeCheckFlag, timeWindowMinutes,
+ singleNodeType, skipIndexUpdateFix );
+ if (fixCandCount == 0) {
+ LOGGER.info(" No fix-Candidates were found by the first pass, so no second/fix-pass is needed. ");
+ } else {
+ // We'll sleep a little and then run a fix-pass based on the
+ // first-run's output file.
+ try {
+ LOGGER.info("About to sleep for " + sleepMinutes
+ + " minutes.");
+ int sleepMsec = sleepMinutes * 60 * 1000;
+ Thread.sleep(sleepMsec);
+ } catch (InterruptedException ie) {
+ LOGGER.info("\n >>> Sleep Thread has been Interrupted <<< ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+
+ dteStr = fd.getDateTime();
+ String secondGroomOutFileName = "dataGrooming." + windowTag + "." + dteStr + ".out";
+ LOGGER.info(" Now, call doTheGrooming() a second time and pass in the name of the file "
+ + "generated by the first pass for fixing: ["
+ + groomOutFileName + "]");
+ finalShutdownFlag = true;
+ cacheDbOkFlag = false;
+ doTheGrooming(groomOutFileName, edgesOnlyFlag,
+ dontFixOrphansFlag, maxRecordsToFix,
+ secondGroomOutFileName, ver, singleCommits,
+ dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
+ finalShutdownFlag, cacheDbOkFlag,
+ skipEdgeCheckFlag, timeWindowMinutes,
+ singleNodeType, skipIndexUpdateFix );
+ }
+ } else {
+ // Do the grooming - plain vanilla (no fix-it-file, no
+ // auto-fixing)
+ Boolean finalShutdownFlag = true;
+ LOGGER.info(" Call doTheGrooming() ");
+ Boolean cacheDbOkFlag = true;
+ if( neverUseCache ){
+ // They have forbidden us from using a cached db connection.
+ cacheDbOkFlag = false;
+ }
+ doTheGrooming("", edgesOnlyFlag, dontFixOrphansFlag,
+ maxRecordsToFix, groomOutFileName, ver, singleCommits,
+ dupeCheckOff, dupeFixOn, ghost2CheckOff, ghost2FixOn,
+ finalShutdownFlag, cacheDbOkFlag,
+ skipEdgeCheckFlag, timeWindowMinutes,
+ singleNodeType, skipIndexUpdateFix );
+ }
+ } catch (Exception ex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("Exception while grooming data " + LogFormatTools.getStackTop(ex));
+ }
+ LOGGER.info(" Done! ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ }
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args) {
+
+ // Set the logging file properties to be used by EELFManager
+ System.setProperty("aai.service.name", DataGrooming.class.getSimpleName());
+
+ LoggingContext.init();
+ LoggingContext.partnerName(FROMAPPID);
+ LoggingContext.serviceName(GraphAdminApp.APP_NAME);
+ LoggingContext.component("dataGrooming");
+ LoggingContext.targetEntity(GraphAdminApp.APP_NAME);
+ LoggingContext.targetServiceName("main");
+ LoggingContext.requestId(TRANSID);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+ DataGrooming dataGrooming = new DataGrooming(loaderFactory, schemaVersions);
+ dataGrooming.execute(args);
+ }
+
+ /**
+ * Do the grooming.
+ *
+ * @param fileNameForFixing the file name for fixing
+ * @param edgesOnlyFlag the edges only flag
+ * @param dontFixOrphansFlag the dont fix orphans flag
+ * @param maxRecordsToFix the max records to fix
+ * @param groomOutFileName the groom out file name
+ * @param version the version
+ * @param singleCommits the single commits
+ * @param dupeCheckOff the dupe check off
+ * @param dupeFixOn the dupe fix on
+ * @param ghost2CheckOff the ghost 2 check off
+ * @param ghost2FixOn the ghost 2 fix on
+ * @param finalShutdownFlag the final shutdown flag
+ * @param cacheDbOkFlag the cacheDbOk flag
+ * @return the int
+ */
+ private int doTheGrooming( String fileNameForFixing,
+ Boolean edgesOnlyFlag, Boolean dontFixOrphansFlag,
+ int maxRecordsToFix, String groomOutFileName, String version,
+ Boolean singleCommits,
+ Boolean dupeCheckOff, Boolean dupeFixOn,
+ Boolean ghost2CheckOff, Boolean ghost2FixOn,
+ Boolean finalShutdownFlag, Boolean cacheDbOkFlag,
+ Boolean skipEdgeCheckFlag, int timeWindowMinutes,
+ String singleNodeType, Boolean skipIndexUpdateFix ) {
+
+ LOGGER.debug(" Entering doTheGrooming \n");
+
+ int cleanupCandidateCount = 0;
+ long windowStartTime = 0; // Translation of the window into a starting timestamp
+ BufferedWriter bw = null;
+ JanusGraph graph = null;
+ JanusGraph graph2 = null;
+ int deleteCount = 0;
+ int dummyUpdCount = 0;
+ boolean executeFinalCommit = false;
+ Set<String> deleteCandidateList = new LinkedHashSet<>();
+ Set<String> processedVertices = new LinkedHashSet<>();
+ Set<String> postCommitRemoveList = new LinkedHashSet<>();
+
+ Graph g = null;
+ Graph g2 = null;
+ try {
+ if( timeWindowMinutes > 0 ){
+ // Translate the window value (ie. 30 minutes) into a unix timestamp like
+ // we use in the db - so we can select data created after that time.
+ windowStartTime = figureWindowStartTime( timeWindowMinutes );
+ }
+
+ AAIConfig.init();
+ String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP
+ + "logs" + AAIConstants.AAI_FILESEP + "data"
+ + AAIConstants.AAI_FILESEP + "dataGrooming";
+
+ // Make sure the target directory exists
+ new File(targetDir).mkdirs();
+
+ if (!fileNameForFixing.equals("")) {
+ deleteCandidateList = getDeleteList(targetDir,
+ fileNameForFixing, edgesOnlyFlag, dontFixOrphansFlag,
+ dupeFixOn);
+ }
+
+ if (deleteCandidateList.size() > maxRecordsToFix) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(" >> WARNING >> Delete candidate list size ("
+ + deleteCandidateList.size()
+ + ") is too big. The maxFix we are using is: "
+ + maxRecordsToFix
+ + ". No candidates will be deleted. ");
+ // Clear out the list so it won't be processed below.
+ deleteCandidateList = new LinkedHashSet<>();
+ }
+
+ String fullOutputFileName = targetDir + AAIConstants.AAI_FILESEP
+ + groomOutFileName;
+ File groomOutFile = new File(fullOutputFileName);
+ try {
+ groomOutFile.createNewFile();
+ } catch (IOException e) {
+ String emsg = " Problem creating output file ["
+ + fullOutputFileName + "], exception=" + e.getMessage();
+ throw new AAIException("AAI_6124", emsg);
+ }
+
+ LOGGER.info(" Will write to " + fullOutputFileName );
+ bw = new BufferedWriter(new FileWriter(groomOutFile.getAbsoluteFile()));
+ ErrorLogHelper.loadProperties();
+
+ LOGGER.info(" ---- NOTE --- about to open graph (takes a little while)--------\n");
+
+ if( cacheDbOkFlag ){
+ // Since we're just reading (not deleting/fixing anything), we can use
+ // a cached connection to the DB
+
+ // -- note JanusGraphFactory has been leaving db connections open
+ //graph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.CACHED_DB_CONFIG).forService(DataGrooming.class.getSimpleName()).withGraphType("cached").buildConfiguration());
+ graph = AAIGraph.getInstance().getGraph();
+ }
+ else {
+ // -- note JanusGraphFactory has been leaving db connections open
+ //graph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(DataGrooming.class.getSimpleName()).withGraphType("realtime1").buildConfiguration());
+ graph = AAIGraph.getInstance().getGraph();
+ }
+ if (graph == null) {
+ String emsg = "null graph object in DataGrooming\n";
+ throw new AAIException("AAI_6101", emsg);
+ }
+
+ LOGGER.debug(" Got the graph object. ");
+
+ g = graph.newTransaction();
+ if (g == null) {
+ String emsg = "null graphTransaction object in DataGrooming\n";
+ throw new AAIException("AAI_6101", emsg);
+ }
+ GraphTraversalSource source1 = g.traversal();
+
+ ArrayList<String> errArr = new ArrayList<>();
+ int totalNodeCount = 0;
+ HashMap<String, String> misMatchedHash = new HashMap<String, String>();
+ HashMap<String, Vertex> orphanNodeHash = new HashMap<String, Vertex>();
+ HashMap<String, Vertex> missingAaiNtNodeHash = new HashMap<String, Vertex>();
+ HashMap<String, Edge> oneArmedEdgeHash = new HashMap<String, Edge>();
+ HashMap<String, String> emptyVertexHash = new HashMap<String, String>();
+ HashMap<String, Vertex> ghostNodeHash = new HashMap<String, Vertex>();
+ ArrayList<String> dupeGroups = new ArrayList<>();
+
+ Loader loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion());
+
+
+ // NOTE --- At one point, we tried explicitly searching for
+ // nodes that were missing their aai-node-type (which does
+ // happen sometimes), but the search takes too long and cannot
+ // be restricted to a date-range since these nodes usually do
+ // not have timestamps either. Instead, when we run across them
+ // as orphans, we will not treat them as orphans, but catagorize
+ // them as "missingAaiNodeType" - which we will treat more like
+ // ghost nodes - that is, delete them without asking permission.
+ //
+ // Note Also - It's a little surprising that we can run
+ // across these when looking for orphans since that search at
+ // least begins based on a given aai-node-type. But watching
+ // where they come up, they are getting discovered when a node
+ // is looking for its parent node. So, say, a “tenant” node
+ // follows a “contains” edge and finds the bad node.
+
+
+
+ Set<Entry<String, Introspector>> entrySet = loader.getAllObjects().entrySet();
+ String ntList = "";
+ LOGGER.info(" Starting DataGrooming Processing ");
+
+ if (edgesOnlyFlag) {
+ LOGGER.info(" NOTE >> Skipping Node processing as requested. Will only process Edges. << ");
+ }
+ else {
+ for (Entry<String, Introspector> entry : entrySet) {
+ String nType = entry.getKey();
+ int thisNtCount = 0;
+ int thisNtDeleteCount = 0;
+
+ if( !singleNodeType.equals("") && !singleNodeType.equals(nType) ){
+ // We are only going to process this one node type
+ continue;
+ }
+
+ LOGGER.debug(" > Look at : [" + nType + "] ...");
+ ntList = ntList + "," + nType;
+
+ // Get a collection of the names of the key properties for this nodeType to use later
+ // Determine what the key fields are for this nodeType - use an arrayList so they
+ // can be gotten out in a consistent order.
+ Set <String> keyPropsSet = entry.getValue().getKeys();
+ ArrayList <String> keyProps = new ArrayList <String> ();
+ keyProps.addAll(keyPropsSet);
+
+ Set <String> indexedPropsSet = entry.getValue().getIndexedProperties();
+ ArrayList <String> indexedProps = new ArrayList <String> ();
+ indexedProps.addAll(indexedPropsSet);
+
+ Iterator<String> indPropItr = indexedProps.iterator();
+ HashMap <String,String> propTypeHash = new HashMap <String, String> ();
+ while( indPropItr.hasNext() ){
+ String propName = indPropItr.next();
+ String propType = entry.getValue().getType(propName);
+ propTypeHash.put(propName, propType);
+ }
+
+ // Get the types of nodes that this nodetype depends on for uniqueness (if any)
+ Collection <String> depNodeTypes = loader.introspectorFromName(nType).getDependentOn();
+
+ // Loop through all the nodes of this Node type
+ int lastShownForNt = 0;
+ ArrayList <Vertex> tmpList = new ArrayList <> ();
+ Iterator <Vertex> iterv = source1.V().has("aai-node-type",nType);
+ while (iterv.hasNext()) {
+ // We put the nodes into an ArrayList because the graph.query iterator can time out
+ tmpList.add(iterv.next());
+ }
+
+ Iterator <Vertex> iter = tmpList.iterator();
+ while (iter.hasNext()) {
+ try {
+ thisNtCount++;
+ if( thisNtCount == lastShownForNt + 1000 ){
+ lastShownForNt = thisNtCount;
+ LOGGER.debug("count for " + nType + " so far = " + thisNtCount );
+ }
+ Vertex thisVtx = iter.next();
+ if( windowStartTime > 0 ){
+ // They are using the time-window, so we only want nodes that are updated after a
+ // passed-in timestamp OR that have no last-modified-timestamp which means they are suspicious.
+ Object objModTimeStamp = thisVtx.property("aai-last-mod-ts").orElse(null);
+ if( objModTimeStamp != null ){
+ long thisNodeModTime = (long)objModTimeStamp;
+ if( thisNodeModTime < windowStartTime ){
+ // It has a last modified ts and is NOT in our window, so we can pass over it
+ continue;
+ }
+ }
+ }
+
+ String thisVid = thisVtx.id().toString();
+ if (processedVertices.contains(thisVid)) {
+ LOGGER.debug("skipping already processed vertex: " + thisVid);
+ continue;
+ }
+ totalNodeCount++;
+ List <Vertex> secondGetList = new ArrayList <> ();
+ // -----------------------------------------------------------------------
+ // For each vertex of this nodeType, we want to:
+ // a) make sure that it can be retrieved using it's AAI defined key
+ // b) make sure that it is not a duplicate
+ // -----------------------------------------------------------------------
+
+ // For this instance of this nodeType, get the key properties
+ HashMap<String, Object> propHashWithKeys = new HashMap<>();
+ Iterator<String> keyPropI = keyProps.iterator();
+ while (keyPropI.hasNext()) {
+ String propName = keyPropI.next();
+ String propVal = "";
+ //delete an already deleted vertex
+ Object obj = thisVtx.<Object>property(propName).orElse(null);
+ if (obj != null) {
+ propVal = obj.toString();
+ }
+ propHashWithKeys.put(propName, propVal);
+ }
+ try {
+ // If this node is dependent on another for uniqueness, then do the query from that parent node
+ // Note - all of our nodes that are dependent on others for uniqueness are
+ // "children" of that node.
+ boolean depNodeOk = true;
+ if( depNodeTypes.isEmpty() ){
+ // This kind of node is not dependent on any other.
+ // Make sure we can get it back using it's key properties (that is the
+ // phantom checking) and that we only get one. Note - we also need
+ // to collect data for a second type of dupe-checking which is done later.
+ secondGetList = getNodeJustUsingKeyParams( TRANSID, FROMAPPID, source1, nType,
+ propHashWithKeys, version );
+ }
+ else {
+ // This kind of node is dependent on another for uniqueness.
+ // Start at it's parent (the parent/containing vertex) and make sure we can get it
+ // back using it's key properties and that we only get one.
+ Iterator <Vertex> vertI2 = source1.V(thisVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
+ Vertex parentVtx = null;
+ // First we need to try to find the parent/containing vertex.
+ int pCount = 0;
+ while( vertI2 != null && vertI2.hasNext() ){
+ parentVtx = vertI2.next();
+ pCount++;
+ }
+ if( pCount <= 0 ){
+ // It's Missing it's dependent/parent/containing node - it's an orphan
+ depNodeOk = false;
+ if (deleteCandidateList.contains(thisVid)) {
+ boolean okFlag = true;
+ boolean updateOnlyFlag = false;
+ try {
+ processedVertices.add(thisVtx.id().toString());
+ Object ob = thisVtx.<Object>property("aai-node-type").orElse(null);
+ if( ob == null && !skipIndexUpdateFix ){
+ updateIndexedProps(thisVtx, thisVid, nType, propTypeHash, indexedProps);
+ updateOnlyFlag = true;
+ dummyUpdCount++;
+ // Since we are updating this delete candidate, not deleting it, we
+ // want it to show up as a delete candidate for this run also.
+ missingAaiNtNodeHash.put(thisVid, thisVtx);
+ }
+ else {
+ // There was an aai-node-type parameter, so we'll do the remove
+ thisVtx.remove();
+ deleteCount++;
+ thisNtDeleteCount++;
+ }
+ } catch (Exception e) {
+ okFlag = false;
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("ERROR trying to delete delete Candidate VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
+ }
+ if (okFlag){
+ if( updateOnlyFlag ) {
+ LOGGER.info(" Updated Indexes for Delete Candidate VID = " + thisVid);
+ }
+ else {
+ LOGGER.info(" DELETED Delete Candidate VID = " + thisVid);
+ }
+ }
+ } else {
+ // NOTE - Only nodes that are missing their parent/containing node are ever considered "orphaned".
+ // That is, you could have a node with no edges... which sounds like an orphan, but not all
+ // nodes require edges. For example, you could have a newly created "image" node which does not have
+ // any edges connected to it (using it) yet.
+ Object ob = thisVtx.<Object>property("aai-node-type").orElse(null);
+ if( ob == null ){
+ // Group this with missing-node-type guys - which
+ // we will delete more readily than orphans.
+ LOGGER.info(" >> Encountered a missingAaiNodeType while looking for the parent of a [" + nType + "] node.");
+ missingAaiNtNodeHash.put(thisVid, thisVtx);
+ }
+ else {
+ Object ob2 = thisVtx.<Object>property("aai-uuid").orElse(null);
+ String auid = "";
+ if( ob2 != null ){
+ auid = ob2.toString();
+ }
+ String checkDummyUid = thisVid + "dummy";
+ if( auid.equals(checkDummyUid) ){
+ // Group this with missing-node-type guys.
+ LOGGER.info(" >> Encountered a missingAaiNodeType mid-fix-node while looking for the parent of a [" + nType + "] node.");
+ missingAaiNtNodeHash.put(thisVid, thisVtx);
+ }
+ else {
+ // It's a regular old orphan
+ orphanNodeHash.put(thisVid, thisVtx);
+ }
+ }
+ }
+ }
+ else if ( pCount > 1 ){
+ // Not sure how this could happen? Should we do something here?
+ depNodeOk = false;
+ }
+ else {
+ // We found the parent - so use it to do the second-look.
+ // NOTE --- We're just going to do the same check from the other direction - because
+ // there could be duplicates or the pointer going the other way could be broken
+ ArrayList <Vertex> tmpListSec = new ArrayList <> ();
+
+ tmpListSec = getConnectedChildrenOfOneType( source1, parentVtx, nType ) ;
+ Iterator<Vertex> vIter = tmpListSec.iterator();
+ while (vIter.hasNext()) {
+ Vertex tmpV = vIter.next();
+ if( vertexHasTheseKeys(tmpV, propHashWithKeys) ){
+ secondGetList.add(tmpV);
+ }
+ }
+ }
+ }// end of -- else this is a dependent node -- piece
+
+ if( depNodeOk && (secondGetList == null || secondGetList.size() == 0) ){
+ // We could not get the node back using it's own key info.
+ // So, it's a PHANTOM
+ if (deleteCandidateList.contains(thisVid)) {
+ boolean okFlag = true;
+ boolean updateOnlyFlag = false;
+ try {
+ Object ob = thisVtx.<Object>property("aai-node-type").orElse(null);
+ if( ob == null && !skipIndexUpdateFix ){
+ updateIndexedProps(thisVtx, thisVid, nType, propTypeHash, indexedProps);
+ dummyUpdCount++;
+ updateOnlyFlag = true;
+ // Since we are updating this delete candidate, not deleting it, we
+ // want it to show up as a delete candidate for this run also.
+ missingAaiNtNodeHash.put(thisVid, thisVtx);
+ }
+ else {
+ // There was an aai-node-type parameter, so we'll do the remove
+ thisVtx.remove();
+ deleteCount++;
+ thisNtDeleteCount++;
+ }
+ } catch (Exception e) {
+ okFlag = false;
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("ERROR trying to delete phantom VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
+ }
+ if (okFlag){
+ if( updateOnlyFlag ) {
+ LOGGER.info(" Updated Indexes for Delete Candidate VID = " + thisVid);
+ }
+ else {
+ LOGGER.info(" DELETED VID = " + thisVid);
+ }
+ }
+ } else {
+ ghostNodeHash.put(thisVid, thisVtx);
+ }
+ }
+ else if( (secondGetList.size() > 1) && depNodeOk && !dupeCheckOff ){
+ // Found some DUPLICATES - need to process them
+ LOGGER.info(" - now check Dupes for this guy - ");
+ List<String> tmpDupeGroups = checkAndProcessDupes(
+ TRANSID, FROMAPPID, g, source1, version,
+ nType, secondGetList, dupeFixOn,
+ deleteCandidateList, singleCommits, dupeGroups, loader);
+ Iterator<String> dIter = tmpDupeGroups.iterator();
+ while (dIter.hasNext()) {
+ // Add in any newly found dupes to our running list
+ String tmpGrp = dIter.next();
+ LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
+ dupeGroups.add(tmpGrp);
+ }
+ }
+ }
+ catch (AAIException e1) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(" For nodeType = " + nType + " Caught exception", e1);
+ errArr.add(e1.getErrorObject().toString());
+ }
+ catch (Exception e2) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(" For nodeType = " + nType
+ + " Caught exception", e2);
+ errArr.add(e2.getMessage());
+ }
+ }// try block to enclose looping over each single vertex
+ catch (Exception exx) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn("WARNING from inside the while-verts-loop ", exx);
+ }
+
+ } // while loop for each record of a nodeType
+
+ if( depNodeTypes.isEmpty() && !dupeCheckOff ){
+ // For this nodeType, we haven't looked at the possibility of a
+ // non-dependent node where two verts have same key info
+ ArrayList<ArrayList<Vertex>> nonDependentDupeSets = new ArrayList<ArrayList<Vertex>>();
+ nonDependentDupeSets = getDupeSets4NonDepNodes(
+ TRANSID, FROMAPPID, g,
+ version, nType, tmpList,
+ keyProps, loader );
+ // For each set found (each set is for a unique instance of key-values),
+ // process the dupes found
+ Iterator<ArrayList<Vertex>> dsItr = nonDependentDupeSets.iterator();
+ while( dsItr.hasNext() ){
+ ArrayList<Vertex> dupeList = dsItr.next();
+ LOGGER.info(" - now check Dupes for some non-dependent guys - ");
+ List<String> tmpDupeGroups = checkAndProcessDupes(
+ TRANSID, FROMAPPID, g, source1, version,
+ nType, dupeList, dupeFixOn,
+ deleteCandidateList, singleCommits, dupeGroups, loader);
+ Iterator<String> dIter = tmpDupeGroups.iterator();
+ while (dIter.hasNext()) {
+ // Add in any newly found dupes to our running list
+ String tmpGrp = dIter.next();
+ LOGGER.info("Found set of dupes: [" + tmpGrp + "]");
+ dupeGroups.add(tmpGrp);
+ }
+ }
+
+ }// end of extra dupe check for non-dependent nodes
+
+ if ( (thisNtDeleteCount > 0) && singleCommits ) {
+ // NOTE - the singleCommits option is not used in normal processing
+ g.tx().commit();
+ g = AAIGraph.getInstance().getGraph().newTransaction();
+
+ }
+ thisNtDeleteCount = 0;
+ LOGGER.info( " Processed " + thisNtCount + " records for [" + nType + "], " + totalNodeCount + " total (in window) overall. " );
+
+ }// While-loop for each node type
+
+ }// end of check to make sure we weren't only supposed to do edges
+
+
+ if( !skipEdgeCheckFlag ){
+ // --------------------------------------------------------------------------------------
+ // Now, we're going to look for one-armed-edges. Ie. an edge that
+ // should have
+ // been deleted (because a vertex on one side was deleted) but
+ // somehow was not deleted.
+ // So the one end of it points to a vertexId -- but that vertex is
+ // empty.
+ // --------------------------------------------------------------------------------------
+
+ // To do some strange checking - we need a second graph object
+ LOGGER.debug(" ---- DEBUG --- about to open a SECOND graph (takes a little while)--------\n");
+ // Note - graph2 just reads - but we want it to use a fresh connection to
+ // the database, so we are NOT using the CACHED DB CONFIG here.
+
+ // -- note JanusGraphFactory has been leaving db connections open
+ //graph2 = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(DataGrooming.class.getSimpleName()).withGraphType("realtime2").buildConfiguration());
+ graph2 = AAIGraph.getInstance().getGraph();
+ if (graph2 == null) {
+ String emsg = "null graph2 object in DataGrooming\n";
+ throw new AAIException("AAI_6101", emsg);
+ } else {
+ LOGGER.debug("Got the graph2 object... \n");
+ }
+ g2 = graph2.newTransaction();
+ if (g2 == null) {
+ String emsg = "null graphTransaction2 object in DataGrooming\n";
+ throw new AAIException("AAI_6101", emsg);
+ }
+
+ ArrayList<Vertex> vertList = new ArrayList<>();
+ Iterator<Vertex> vItor3 = g.traversal().V();
+ // Gotta hold these in a List - or else HBase times out as you cycle
+ // through these
+ while (vItor3.hasNext()) {
+ Vertex v = vItor3.next();
+ vertList.add(v);
+ }
+ int counter = 0;
+ int lastShown = 0;
+ Iterator<Vertex> vItor2 = vertList.iterator();
+ LOGGER.info(" Checking for bad edges --- ");
+
+ while (vItor2.hasNext()) {
+ Vertex v = null;
+ try {
+ try {
+ v = vItor2.next();
+ } catch (Exception vex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(">>> WARNING trying to get next vertex on the vItor2 ");
+ continue;
+ }
+
+ counter++;
+ String thisVertId = "";
+ try {
+ thisVertId = v.id().toString();
+ } catch (Exception ev) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn("WARNING when doing getId() on a vertex from our vertex list. ");
+ continue;
+ }
+ if (ghostNodeHash.containsKey(thisVertId)) {
+ // This is a phantom node, so don't try to use it
+ LOGGER.info(" >> Skipping edge check for edges from vertexId = "
+ + thisVertId
+ + ", since that guy is a Phantom Node");
+ continue;
+ }
+
+ if( windowStartTime > 0 ){
+ // They are using the time-window, so we only want nodes that are updated after a
+ // passed-in timestamp OR that have no last-modified-timestamp which means they are suspicious.
+ Object objModTimeStamp = v.property("aai-last-mod-ts").orElse(null);
+ if( objModTimeStamp != null ){
+ long thisNodeModTime = (long)objModTimeStamp;
+ if( thisNodeModTime < windowStartTime ){
+ // It has a last modified ts and is NOT in our window, so we can pass over it
+ continue;
+ }
+ }
+ }
+
+ if (counter == lastShown + 250) {
+ lastShown = counter;
+ LOGGER.info("... Checking edges for vertex # "
+ + counter);
+ }
+ Iterator<Edge> eItor = v.edges(Direction.BOTH);
+ while (eItor.hasNext()) {
+ Edge e = null;
+ Vertex vIn = null;
+ Vertex vOut = null;
+ try {
+ e = eItor.next();
+ } catch (Exception iex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(">>> WARNING trying to get next edge on the eItor ", iex);
+ continue;
+ }
+
+ try {
+ vIn = e.inVertex();
+ } catch (Exception err) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(">>> WARNING trying to get edge's In-vertex ", err);
+ }
+ String vNtI = "";
+ String vIdI = "";
+ Vertex ghost2 = null;
+
+ Boolean keysMissing = true;
+ Boolean cantGetUsingVid = false;
+ if (vIn != null) {
+ try {
+ Object ob = vIn.<Object>property("aai-node-type").orElse(null);
+ if (ob != null) {
+ vNtI = ob.toString();
+ keysMissing = anyKeyFieldsMissing(vNtI, vIn, loader);
+ }
+ ob = vIn.id();
+ long vIdLong = 0L;
+ if (ob != null) {
+ vIdI = ob.toString();
+ vIdLong = Long.parseLong(vIdI);
+ }
+
+ if( ! ghost2CheckOff ){
+ Vertex connectedVert = g2.traversal().V(vIdLong).next();
+ if( connectedVert == null ) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
+ cantGetUsingVid = true;
+
+ // If we can NOT get this ghost with the SECOND graph-object,
+ // it is still a ghost since even though we can get data about it using the FIRST graph
+ // object.
+ try {
+ ghost2 = g.traversal().V(vIdLong).next();
+ }
+ catch( Exception ex){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
+ }
+ if( ghost2 != null ){
+ ghostNodeHash.put(vIdI, ghost2);
+ }
+ }
+ }// end of the ghost2 checking
+ }
+ catch (Exception err) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(">>> WARNING trying to get edge's In-vertex props ", err);
+ }
+ }
+ if (keysMissing || vIn == null || vNtI.equals("")
+ || cantGetUsingVid) {
+ // this is a bad edge because it points to a vertex
+ // that isn't there anymore or is corrupted
+ String thisEid = e.id().toString();
+ if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdI)) {
+ boolean okFlag = true;
+ if (!vIdI.equals("")) {
+ // try to get rid of the corrupted vertex
+ try {
+ if( (ghost2 != null) && ghost2FixOn ){
+ ghost2.remove();
+ }
+ else {
+ vIn.remove();
+ }
+ if (singleCommits) {
+ // NOTE - the singleCommits option is not used in normal processing
+ g.tx().commit();
+ g = AAIGraph.getInstance().getGraph().newTransaction();
+ }
+ else {
+ executeFinalCommit = true;
+ }
+ deleteCount++;
+ } catch (Exception e1) {
+ okFlag = false;
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn("WARNING when trying to delete bad-edge-connected VERTEX VID = "
+ + vIdI, e1);
+ }
+ if (okFlag) {
+ LOGGER.info(" DELETED vertex from bad edge = "
+ + vIdI);
+ }
+ } else {
+ // remove the edge if we couldn't get the
+ // vertex
+ try {
+ e.remove();
+ if (singleCommits) {
+ // NOTE - the singleCommits option is not used in normal processing
+ g.tx().commit();
+ g = AAIGraph.getInstance().getGraph().newTransaction();
+ }
+ else {
+ executeFinalCommit = true;
+ }
+ deleteCount++;
+ } catch (Exception ex) {
+ // NOTE - often, the exception is just
+ // that this edge has already been
+ // removed
+ okFlag = false;
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn("WARNING when trying to delete edge = "
+ + thisEid);
+ }
+ if (okFlag) {
+ LOGGER.info(" DELETED edge = " + thisEid);
+ }
+ }
+ } else {
+ oneArmedEdgeHash.put(thisEid, e);
+ if ((vIn != null) && (vIn.id() != null)) {
+ emptyVertexHash.put(thisEid, vIn.id()
+ .toString());
+ }
+ }
+ }
+
+ try {
+ vOut = e.outVertex();
+ } catch (Exception err) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(">>> WARNING trying to get edge's Out-vertex ");
+ }
+ String vNtO = "";
+ String vIdO = "";
+ ghost2 = null;
+ keysMissing = true;
+ cantGetUsingVid = false;
+ if (vOut != null) {
+ try {
+ Object ob = vOut.<Object>property("aai-node-type").orElse(null);
+ if (ob != null) {
+ vNtO = ob.toString();
+ keysMissing = anyKeyFieldsMissing(vNtO,
+ vOut, loader);
+ }
+ ob = vOut.id();
+ long vIdLong = 0L;
+ if (ob != null) {
+ vIdO = ob.toString();
+ vIdLong = Long.parseLong(vIdO);
+ }
+
+ if( ! ghost2CheckOff ){
+ Vertex connectedVert = g2.traversal().V(vIdLong).next();
+ if( connectedVert == null ) {
+ cantGetUsingVid = true;
+ LOGGER.info( "GHOST2 -- got NULL when doing getVertex for vid = " + vIdLong);
+ // If we can get this ghost with the other graph-object, then get it -- it's still a ghost
+ try {
+ ghost2 = g.traversal().V(vIdLong).next();
+ }
+ catch( Exception ex){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn( "GHOST2 -- Could not get the ghost info for a bad edge for vtxId = " + vIdLong, ex);
+ }
+ if( ghost2 != null ){
+ ghostNodeHash.put(vIdO, ghost2);
+ }
+ }
+ }
+ } catch (Exception err) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(">>> WARNING trying to get edge's Out-vertex props ", err);
+ }
+ }
+ if (keysMissing || vOut == null || vNtO.equals("")
+ || cantGetUsingVid) {
+ // this is a bad edge because it points to a vertex
+ // that isn't there anymore
+ String thisEid = e.id().toString();
+ if (deleteCandidateList.contains(thisEid) || deleteCandidateList.contains(vIdO)) {
+ boolean okFlag = true;
+ if (!vIdO.equals("")) {
+ // try to get rid of the corrupted vertex
+ try {
+ if( (ghost2 != null) && ghost2FixOn ){
+ ghost2.remove();
+ }
+ else if (vOut != null) {
+ vOut.remove();
+ }
+ if (singleCommits) {
+ // NOTE - the singleCommits option is not used in normal processing
+ g.tx().commit();
+ g = AAIGraph.getInstance().getGraph().newTransaction();
+ }
+ else {
+ executeFinalCommit = true;
+ }
+ deleteCount++;
+ } catch (Exception e1) {
+ okFlag = false;
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn("WARNING when trying to delete bad-edge-connected VID = "
+ + vIdO, e1);
+ }
+ if (okFlag) {
+ LOGGER.info(" DELETED vertex from bad edge = "
+ + vIdO);
+ }
+ } else {
+ // remove the edge if we couldn't get the
+ // vertex
+ try {
+ e.remove();
+ if (singleCommits) {
+ // NOTE - the singleCommits option is not used in normal processing
+ g.tx().commit();
+ g = AAIGraph.getInstance().getGraph().newTransaction();
+ }
+ else {
+ executeFinalCommit = true;
+ }
+ deleteCount++;
+ } catch (Exception ex) {
+ // NOTE - often, the exception is just
+ // that this edge has already been
+ // removed
+ okFlag = false;
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn("WARNING when trying to delete edge = "
+ + thisEid, ex);
+ }
+ if (okFlag) {
+ LOGGER.info(" DELETED edge = " + thisEid);
+ }
+ }
+ } else {
+ oneArmedEdgeHash.put(thisEid, e);
+ if ((vOut != null) && (vOut.id() != null)) {
+ emptyVertexHash.put(thisEid, vOut.id()
+ .toString());
+ }
+ }
+ }
+ }// End of while-edges-loop
+ } catch (Exception exx) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn("WARNING from in the while-verts-loop ", exx);
+ }
+ }// End of while-vertices-loop (the edge-checking)
+ LOGGER.info(" Done checking for bad edges --- ");
+ } // end of -- if we're not skipping the edge-checking
+
+
+ deleteCount = deleteCount + dupeGrpsDeleted;
+ if (!singleCommits && (deleteCount > 0 || dummyUpdCount > 0) ){
+ executeFinalCommit = true;
+ }
+
+ int ghostNodeCount = ghostNodeHash.size();
+ int orphanNodeCount = orphanNodeHash.size();
+ int oneArmedEdgeCount = oneArmedEdgeHash.size();
+ int missingAaiNtNodeCount = missingAaiNtNodeHash.size();
+ int dupeCount = dupeGroups.size();
+
+ deleteCount = deleteCount + dupeGrpsDeleted;
+
+ bw.write("\n\n ============ Summary ==============\n");
+ if( timeWindowMinutes == 0 ){
+ bw.write("Ran FULL data grooming (no time-window). \n");
+ }
+ else {
+ bw.write("Ran PARTIAL data grooming just looking at data added/updated in the last " + timeWindowMinutes + " minutes. \n");
+ }
+
+ bw.write("\nRan these nodeTypes: " + ntList + "\n\n");
+ bw.write("There were this many delete candidates from previous run = "
+ + deleteCandidateList.size() + "\n");
+ if (dontFixOrphansFlag) {
+ bw.write(" Note - we are not counting orphan nodes since the -dontFixOrphans parameter was used. \n");
+ }
+ bw.write("Deleted this many delete candidates = " + deleteCount
+ + "\n");
+ bw.write("Dummy-index-update to delete candidates = " + dummyUpdCount
+ + "\n");
+ bw.write("Total number of nodes looked at = " + totalNodeCount
+ + "\n");
+ bw.write("Ghost Nodes identified = " + ghostNodeCount + "\n");
+ bw.write("Orphan Nodes identified = " + orphanNodeCount + "\n");
+ bw.write("Missing aai-node-type Nodes identified = " + missingAaiNtNodeCount + "\n");
+ bw.write("Bad Edges identified = " + oneArmedEdgeCount + "\n");
+ bw.write("Duplicate Groups count = " + dupeCount + "\n");
+ bw.write("MisMatching Label/aai-node-type count = "
+ + misMatchedHash.size() + "\n");
+
+ bw.write("\n ------------- Delete Candidates ---------\n");
+ for (Map.Entry<String, Vertex> entry : ghostNodeHash
+ .entrySet()) {
+ String vid = entry.getKey();
+ bw.write("DeleteCandidate: Phantom Vid = [" + vid + "]\n");
+ cleanupCandidateCount++;
+ }
+ for (Map.Entry<String, Vertex> entry : missingAaiNtNodeHash
+ .entrySet()) {
+ String vid = entry.getKey();
+ bw.write("DeleteCandidate: Missing aai-node-type Vid = [" + vid + "]\n");
+ cleanupCandidateCount++;
+ }
+ for (Map.Entry<String, Vertex> entry : orphanNodeHash
+ .entrySet()) {
+ String vid = entry.getKey();
+ bw.write("DeleteCandidate: OrphanDepNode Vid = [" + vid + "]\n");
+ if (!dontFixOrphansFlag) {
+ cleanupCandidateCount++;
+ }
+ }
+ for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
+ String eid = entry.getKey();
+ bw.write("DeleteCandidate: Bad EDGE Edge-id = [" + eid + "]\n");
+ cleanupCandidateCount++;
+ }
+
+ bw.write("\n-- NOTE - To see DeleteCandidates for Duplicates, you need to look in the Duplicates Detail section below.\n");
+
+ bw.write("\n ------------- GHOST NODES - detail ");
+ for (Map.Entry<String, Vertex> entry : ghostNodeHash
+ .entrySet()) {
+ try {
+ String vid = entry.getKey();
+ bw.write("\n ==> Phantom Vid = " + vid + "\n");
+ ArrayList<String> retArr = showPropertiesForNode(
+ TRANSID, FROMAPPID, entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+ retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
+ entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+ } catch (Exception dex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("error trying to print detail info for a ghost-node: " + LogFormatTools.getStackTop(dex));
+ }
+ }
+
+ bw.write("\n ------------- Missing aai-node-type NODES - detail: ");
+ for (Map.Entry<String, Vertex> entry : missingAaiNtNodeHash
+ .entrySet()) {
+ try {
+ String vid = entry.getKey();
+ bw.write("\n> Missing aai-node-type Node Vid = " + vid + "\n");
+ ArrayList<String> retArr = showPropertiesForNode(
+ TRANSID, FROMAPPID, entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+
+ retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
+ entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+ } catch (Exception dex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("error trying to print detail info for a node missing its aai-node-type " + LogFormatTools.getStackTop(dex));
+ }
+ }
+
+ bw.write("\n ------------- Missing Dependent Edge ORPHAN NODES - detail: ");
+ for (Map.Entry<String, Vertex> entry : orphanNodeHash
+ .entrySet()) {
+ try {
+ String vid = entry.getKey();
+ bw.write("\n> Orphan Node Vid = " + vid + "\n");
+ ArrayList<String> retArr = showPropertiesForNode(
+ TRANSID, FROMAPPID, entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+
+ retArr = showAllEdgesForNode(TRANSID, FROMAPPID,
+ entry.getValue());
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+ } catch (Exception dex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("error trying to print detail info for a Orphan Node /missing dependent edge " + LogFormatTools.getStackTop(dex));
+ }
+ }
+
+ bw.write("\n ------------- EDGES pointing to empty/bad vertices: ");
+ for (Map.Entry<String, Edge> entry : oneArmedEdgeHash.entrySet()) {
+ try {
+ String eid = entry.getKey();
+ Edge thisE = entry.getValue();
+ String badVid = emptyVertexHash.get(eid);
+ bw.write("\n> Edge pointing to bad vertex (Vid = "
+ + badVid + ") EdgeId = " + eid + "\n");
+ bw.write("Label: [" + thisE.label() + "]\n");
+ Iterator<Property<Object>> pI = thisE.properties();
+ while (pI.hasNext()) {
+ Property<Object> propKey = pI.next();
+ bw.write("Prop: [" + propKey + "], val = ["
+ + propKey.value() + "]\n");
+ }
+ } catch (Exception pex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("error trying to print empty/bad vertex data: " + LogFormatTools.getStackTop(pex));
+ }
+ }
+
+ bw.write("\n ------------- Duplicates: ");
+ Iterator<String> dupeIter = dupeGroups.iterator();
+ int dupeSetCounter = 0;
+ while (dupeIter.hasNext()) {
+ dupeSetCounter++;
+ String dset = (String) dupeIter.next();
+
+ bw.write("\n --- Duplicate Group # " + dupeSetCounter
+ + " Detail -----------\n");
+ try {
+ // We expect each line to have at least two vid's, followed
+ // by the preferred one to KEEP
+ String[] dupeArr = dset.split("\\|");
+ ArrayList<String> idArr = new ArrayList<>();
+ int lastIndex = dupeArr.length - 1;
+ for (int i = 0; i <= lastIndex; i++) {
+ if (i < lastIndex) {
+ // This is not the last entry, it is one of the
+ // dupes, so we want to show all its info
+ bw.write(" >> Duplicate Group # "
+ + dupeSetCounter + " Node # " + i
+ + " ----\n");
+ String vidString = dupeArr[i];
+ idArr.add(vidString);
+ long longVertId = Long.parseLong(vidString);
+ Iterator<Vertex> vtxIterator = g.vertices(longVertId);
+ Vertex vtx = null;
+ if (vtxIterator.hasNext()) {
+ vtx = vtxIterator.next();
+ }
+ ArrayList<String> retArr = showPropertiesForNode(TRANSID, FROMAPPID, vtx);
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+
+ retArr = showAllEdgesForNode(TRANSID,
+ FROMAPPID, vtx);
+ for (String info : retArr) {
+ bw.write(info + "\n");
+ }
+ } else {
+ // This is the last entry which should tell us if we
+ // have a preferred keeper
+ String prefString = dupeArr[i];
+ if (prefString.equals("KeepVid=UNDETERMINED")) {
+ bw.write("\n For this group of duplicates, could not tell which one to keep.\n");
+ bw.write(" >>> This group needs to be taken care of with a manual/forced-delete.\n");
+ } else {
+ // If we know which to keep, then the prefString
+ // should look like, "KeepVid=12345"
+ String[] prefArr = prefString.split("=");
+ if (prefArr.length != 2
+ || (!prefArr[0].equals("KeepVid"))) {
+ throw new Exception("Bad format. Expecting KeepVid=999999");
+ } else {
+ String keepVidStr = prefArr[1];
+ if (idArr.contains(keepVidStr)) {
+ bw.write("\n The vertex we want to KEEP has vertexId = "
+ + keepVidStr);
+ bw.write("\n The others become delete candidates: \n");
+ idArr.remove(keepVidStr);
+ for (int x = 0; x < idArr.size(); x++) {
+ cleanupCandidateCount++;
+ bw.write("DeleteCandidate: Duplicate Vid = ["
+ + idArr.get(x) + "]\n");
+ }
+ } else {
+ throw new Exception("ERROR - Vertex Id to keep not found in list of dupes. dset = ["
+ + dset + "]");
+ }
+ }
+ }// else we know which one to keep
+ }// else last entry
+ }// for each vertex in a group
+ } catch (Exception dex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("error trying to print duplicate vertex data " + LogFormatTools.getStackTop(dex));
+ }
+
+ }// while - work on each group of dupes
+
+ bw.write("\n ------------- Mis-matched Label/aai-node-type Nodes: \n ");
+ for (Map.Entry<String, String> entry : misMatchedHash.entrySet()) {
+ String msg = entry.getValue();
+ bw.write("MixedMsg = " + msg + "\n");
+ }
+
+ bw.write("\n ------------- Got these errors while processing: \n");
+ Iterator<String> errIter = errArr.iterator();
+ while (errIter.hasNext()) {
+ String line = (String) errIter.next();
+ bw.write(line + "\n");
+ }
+
+ bw.close();
+
+ LOGGER.info("\n ------------- Done doing all the checks ------------ ");
+ LOGGER.info("Output will be written to " + fullOutputFileName);
+
+ if (cleanupCandidateCount > 0) {
+ // Technically, this is not an error -- but we're throwing this
+ // error so that hopefully a
+ // monitoring system will pick it up and do something with it.
+ throw new AAIException("AAI_6123", "See file: [" + fullOutputFileName
+ + "] and investigate delete candidates. ");
+ }
+ } catch (AAIException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("Caught AAIException while grooming data");
+ ErrorLogHelper.logException(e);
+ } catch (Exception ex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("Caught exception while grooming data");
+ ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun dataGrooming");
+ } finally {
+
+ if (bw != null) {
+ try {
+ bw.close();
+ } catch (IOException iox) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ LOGGER.warn("Got an IOException trying to close bufferedWriter() \n", iox);
+ }
+ }
+
+ if (executeFinalCommit) {
+ // If we were holding off on commits till the end - then now is the time.
+ if( g == null ){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error(" >>>> ERROR <<<< Could not commit changes. graph was null when we wanted to commit.");
+ }
+ else if( !g.tx().isOpen() ){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error(" >>>> ERROR <<<< Could not commit changes. Transaction was not open when we wanted to commit.");
+ }
+ else {
+ try {
+ LOGGER.info("About to do the commit for "
+ + deleteCount + " removes. ");
+ g.tx().commit();
+ LOGGER.info("Commit was successful ");
+ } catch (Exception excom) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error(" >>>> ERROR <<<< Could not commit changes. " + LogFormatTools.getStackTop(excom));
+ deleteCount = 0;
+ }
+ }
+ }
+ else if (g != null && g.tx().isOpen()) {
+ try {
+ // We did not do any deletes that need to be committed.
+ // The rollback is to clear out the transaction used while doing those reads
+ g.tx().rollback();
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ LOGGER.warn("WARNING from final graphTransaction.rollback()", ex);
+ }
+ }
+
+ if (g2 != null && g2.tx().isOpen()) {
+ try {
+ // We only read on g2. The rollback is to clear out the transaction used while doing those reads
+ g2.tx().rollback();
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ LOGGER.warn("WARNING from final graphTransaction2.rollback()", ex);
+ }
+ }
+
+ if( finalShutdownFlag ){
+ try {
+ if( graph != null && graph.isOpen() ){
+ graph.tx().close();
+ if( "true".equals(System.getProperty("org.onap.aai.graphadmin.started"))) {
+ // Since dataGrooming was called from a scheduled task - do not call graph.close()
+ }
+ else {
+ // DataGrooming must have been called manually - so we need to call close().
+ graph.close();
+ }
+ }
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ LOGGER.warn("WARNING from final graph.shutdown()", ex);
+ }
+
+ try {
+ if( graph2 != null && graph2.isOpen() ){
+ graph2.tx().close();
+ if( "true".equals(System.getProperty("org.onap.aai.graphadmin.started"))) {
+ // Since dataGrooming was called from a scheduled task - do not call graph2.close()
+ }
+ else {
+ // DataGrooming must have been called manually - so we need to call close().
+ graph2.close();
+ }
+ }
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ LOGGER.warn("WARNING from final graph2.shutdown()", ex);
+ }
+ }
+
+ }
+
+ return cleanupCandidateCount;
+
+ }// end of doTheGrooming()
+
+
+ private void updateIndexedProps(Vertex thisVtx, String thisVidStr, String nType,
+ HashMap <String,String>propTypeHash, ArrayList <String> indexedProps) {
+ // This is a "missing-aai-node-type" scenario.
+ // Other indexes may also be messed up, so we will update all of them on
+ // this pass. A future pass will just treat this node like a regular orphan
+ // and delete it (if appropriate).
+ LOGGER.info(" We will be updating the indexed properties for this node to dummy values. VID = " + thisVidStr );
+ String dummyPropValStr = thisVidStr + "dummy";
+ // These reserved-prop-names are all indexed for all nodes
+ thisVtx.property("aai-node-type",nType);
+ thisVtx.property("aai-uri", dummyPropValStr);
+ thisVtx.property("aai-unique-key", dummyPropValStr);
+ thisVtx.property("aai-uuid", dummyPropValStr);
+ thisVtx.property("source-of-truth", dummyPropValStr);
+ Iterator<String> indexedPropI = indexedProps.iterator();
+ while (indexedPropI.hasNext()) {
+ String propName = indexedPropI.next();
+ // Using the VID in case this property is unique in the db and
+ // we're doing this kind of thing on more than one of these nodes..
+ String dataType = propTypeHash.get(propName);
+ if( dataType == null || dataType.toLowerCase().endsWith(".string") ){
+ thisVtx.property(propName, dummyPropValStr);
+ }
+ else if( dataType.toLowerCase().endsWith(".long") ){
+ Long thisVidLong = (Long) thisVtx.id();
+ thisVtx.property(propName, thisVidLong);
+ }
+ else if( dataType.toLowerCase().endsWith(".boolean") ){
+ thisVtx.property(propName, false);
+ }
+ else if( dataType.toLowerCase().endsWith(".integer") ){
+ thisVtx.property(propName, 9999);
+ }
+ else {
+ // Not sure what it is - try a string
+ thisVtx.property(propName, dummyPropValStr);
+ }
+ }
+ }
+
+ /**
+ * Vertex has these keys.
+ *
+ * @param tmpV the tmp V
+ * @param propHashWithKeys the prop hash with keys
+ * @return the boolean
+ */
+ private Boolean vertexHasTheseKeys( Vertex tmpV, HashMap <String, Object> propHashWithKeys) {
+ Iterator <?> it = propHashWithKeys.entrySet().iterator();
+ while( it.hasNext() ){
+ String propName = "";
+ String propVal = "";
+ Map.Entry <?,?>propEntry = (Map.Entry<?,?>)it.next();
+ Object propNameObj = propEntry.getKey();
+ if( propNameObj != null ){
+ propName = propNameObj.toString();
+ }
+ Object propValObj = propEntry.getValue();
+ if( propValObj != null ){
+ propVal = propValObj.toString();
+ }
+ Object checkValObj = tmpV.<Object>property(propName).orElse(null);
+ if( checkValObj == null ) {
+ return false;
+ }
+ else if( !propVal.equals(checkValObj.toString()) ){
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ /**
+ * Any key fields missing.
+ *
+ * @param nType the n type
+ * @param v the v
+ * @return the boolean
+ */
+ private Boolean anyKeyFieldsMissing(String nType, Vertex v, Loader loader) {
+
+ try {
+ Introspector obj = null;
+ try {
+ obj = loader.introspectorFromName(nType);
+ } catch (AAIUnknownObjectException e) {
+ // They gave us a non-empty nodeType but our NodeKeyProps does
+ // not have data for it. Since we do not know what the
+ // key params are for this type of node, we will just
+ // return "false".
+ String emsg = " -- WARNING -- Unrecognized nodeType: [" + nType
+ + "]. We cannot determine required keys for this nType. ";
+ // NOTE - this will be caught below and a "false" returned
+ throw new AAIException("AAI_6121", emsg);
+ }
+
+ // Determine what the key fields are for this nodeType
+ Collection <String> keyPropNamesColl = obj.getKeys();
+ Iterator<String> keyPropI = keyPropNamesColl.iterator();
+ while (keyPropI.hasNext()) {
+ String propName = keyPropI.next();
+ Object ob = v.<Object>property(propName).orElse(null);
+ if (ob == null || ob.toString().equals("")) {
+ // It is missing a key property
+ return true;
+ }
+ }
+ } catch (AAIException e) {
+ // Something was wrong -- but since we weren't able to check
+ // the keys, we will not declare that it is missing keys.
+ return false;
+ }
+ return false;
+ }
+
+
+ /**
+ * Gets the delete list.
+ *
+ * @param targetDir the target dir
+ * @param fileName the file name
+ * @param edgesOnlyFlag the edges only flag
+ * @param dontFixOrphans the dont fix orphans
+ * @param dupeFixOn the dupe fix on
+ * @return the delete list
+ * @throws AAIException the AAI exception
+ */
+ private Set<String> getDeleteList(String targetDir,
+ String fileName, Boolean edgesOnlyFlag, Boolean dontFixOrphans,
+ Boolean dupeFixOn) throws AAIException {
+
+ // Look in the file for lines formated like we expect - pull out any
+ // Vertex Id's to delete on this run
+ Set<String> delList = new LinkedHashSet<>();
+ String fullFileName = targetDir + AAIConstants.AAI_FILESEP + fileName;
+
+ try(BufferedReader br = new BufferedReader(new FileReader(fullFileName))) {
+ String line = br.readLine();
+ while (line != null) {
+ if (!"".equals(line) && line.startsWith("DeleteCandidate")) {
+ if (edgesOnlyFlag && (!line.contains("Bad Edge"))) {
+ // We're only processing edges and this line is not for an edge
+ } else if (dontFixOrphans && line.contains("Orphan")) {
+ // We're not going to process orphans
+ } else if (!dupeFixOn && line.contains("Duplicate")) {
+ // We're not going to process Duplicates
+ } else {
+ int begIndex = line.indexOf("id = ");
+ int endIndex = line.indexOf("]");
+ String vidVal = line.substring(begIndex + 6, endIndex);
+ delList.add(vidVal);
+ }
+ }
+ line = br.readLine();
+ }
+ br.close();
+ } catch (IOException e) {
+ throw new AAIException("AAI_6124", e, "Could not open input-file [" + fullFileName
+ + "], exception= " + e.getMessage());
+ }
+
+ return delList;
+
+ }// end of getDeleteList
+
+ /**
+ * Gets the preferred dupe.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param dupeVertexList the dupe vertex list
+ * @param ver the ver
+ * @return Vertex
+ * @throws AAIException the AAI exception
+ */
+ public Vertex getPreferredDupe(String transId,
+ String fromAppId, GraphTraversalSource g,
+ ArrayList<Vertex> dupeVertexList, String ver, Loader loader)
+ throws AAIException {
+
+ // This method assumes that it is being passed a List of vertex objects
+ // which
+ // violate our uniqueness constraints.
+
+ Vertex nullVtx = null;
+
+ if (dupeVertexList == null) {
+ return nullVtx;
+ }
+ int listSize = dupeVertexList.size();
+ if (listSize == 0) {
+ return nullVtx;
+ }
+ if (listSize == 1) {
+ return (dupeVertexList.get(0));
+ }
+
+ Vertex vtxPreferred = null;
+ Vertex currentFaveVtx = dupeVertexList.get(0);
+ for (int i = 1; i < listSize; i++) {
+ Vertex vtxB = dupeVertexList.get(i);
+ vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
+ currentFaveVtx, vtxB, ver, loader);
+ if (vtxPreferred == null) {
+ // We couldn't choose one
+ return nullVtx;
+ } else {
+ currentFaveVtx = vtxPreferred;
+ }
+ }
+
+ return (currentFaveVtx);
+
+ } // end of getPreferredDupe()
+
+ /**
+ * Pick one of two dupes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param vtxA the vtx A
+ * @param vtxB the vtx B
+ * @param ver the ver
+ * @return Vertex
+ * @throws AAIException the AAI exception
+ */
+ public Vertex pickOneOfTwoDupes(String transId,
+ String fromAppId, GraphTraversalSource g, Vertex vtxA,
+ Vertex vtxB, String ver, Loader loader) throws AAIException {
+
+ Vertex nullVtx = null;
+ Vertex preferredVtx = null;
+
+ Long vidA = new Long(vtxA.id().toString());
+ Long vidB = new Long(vtxB.id().toString());
+
+ String vtxANodeType = "";
+ String vtxBNodeType = "";
+ Object objType = vtxA.<Object>property("aai-node-type").orElse(null);
+ if (objType != null) {
+ vtxANodeType = objType.toString();
+ }
+ objType = vtxB.<Object>property("aai-node-type").orElse(null);
+ if (objType != null) {
+ vtxBNodeType = objType.toString();
+ }
+
+ if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
+ // Either they're not really dupes or there's some bad data - so
+ // don't pick one
+ return nullVtx;
+ }
+
+ // Check that node A and B both have the same key values (or else they
+ // are not dupes)
+ // (We'll check dep-node later)
+ // Determine what the key fields are for this nodeType
+ Collection <String> keyProps = new ArrayList <>();
+ HashMap <String,Object> keyPropValsHash = new HashMap <String,Object>();
+ try {
+ keyProps = loader.introspectorFromName(vtxANodeType).getKeys();
+ } catch (AAIUnknownObjectException e) {
+ LOGGER.warn("Required property not found", e);
+ throw new AAIException("AAI_6105", "Required Property name(s) not found for nodeType = " + vtxANodeType + ")");
+ }
+
+ Iterator<String> keyPropI = keyProps.iterator();
+ while (keyPropI.hasNext()) {
+ String propName = keyPropI.next();
+ String vtxAKeyPropVal = "";
+ objType = vtxA.<Object>property(propName).orElse(null);
+ if (objType != null) {
+ vtxAKeyPropVal = objType.toString();
+ }
+ String vtxBKeyPropVal = "";
+ objType = vtxB.<Object>property(propName).orElse(null);
+ if (objType != null) {
+ vtxBKeyPropVal = objType.toString();
+ }
+
+ if (vtxAKeyPropVal.equals("")
+ || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
+ // Either they're not really dupes or they are missing some key
+ // data - so don't pick one
+ return nullVtx;
+ }
+ else {
+ // Keep these around for (potential) use later
+ keyPropValsHash.put(propName, vtxAKeyPropVal);
+ }
+
+ }
+
+ // Collect the vid's and aai-node-types of the vertices that each vertex
+ // (A and B) is connected to.
+ ArrayList<String> vtxIdsConn2A = new ArrayList<>();
+ ArrayList<String> vtxIdsConn2B = new ArrayList<>();
+ HashMap<String, String> nodeTypesConn2A = new HashMap<>();
+ HashMap<String, String> nodeTypesConn2B = new HashMap<>();
+
+ ArrayList<Vertex> vertListA = getConnectedNodes( g, vtxA );
+ if (vertListA != null) {
+ Iterator<Vertex> iter = vertListA.iterator();
+ while (iter.hasNext()) {
+ Vertex tvCon = iter.next();
+ String conVid = tvCon.id().toString();
+ String nt = "";
+ objType = tvCon.<Object>property("aai-node-type").orElse(null);
+ if (objType != null) {
+ nt = objType.toString();
+ }
+ nodeTypesConn2A.put(nt, conVid);
+ vtxIdsConn2A.add(conVid);
+ }
+ }
+
+ ArrayList<Vertex> vertListB = getConnectedNodes( g, vtxB );
+ if (vertListB != null) {
+ Iterator<Vertex> iter = vertListB.iterator();
+ while (iter.hasNext()) {
+ Vertex tvCon = iter.next();
+ String conVid = tvCon.id().toString();
+ String nt = "";
+ objType = tvCon.<Object>property("aai-node-type").orElse(null);
+ if (objType != null) {
+ nt = objType.toString();
+ }
+ nodeTypesConn2B.put(nt, conVid);
+ vtxIdsConn2B.add(conVid);
+ }
+ }
+
+ // 1 - If this kind of node needs a dependent node for uniqueness, then
+ // verify that they both nodes point to the same dependent
+ // node (otherwise they're not really duplicates)
+ // Note - there are sometimes more than one dependent node type since
+ // one nodeType can be used in different ways. But for a
+ // particular node, it will only have one dependent node that
+ // it's connected to.
+ String onlyNodeThatIndexPointsToVidStr = "";
+ Collection<String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn();
+ if (depNodeTypes.isEmpty()) {
+ // This kind of node is not dependent on any other. That is ok.
+ // We need to find out if the unique index info is good or not and
+ // use that later when deciding if we can delete one.
+ onlyNodeThatIndexPointsToVidStr = findJustOneUsingIndex( transId,
+ fromAppId, g, keyPropValsHash, vtxANodeType, vidA, vidB, ver );
+ } else {
+ String depNodeVtxId4A = "";
+ String depNodeVtxId4B = "";
+ Iterator<String> iter = depNodeTypes.iterator();
+ while (iter.hasNext()) {
+ String depNodeType = iter.next();
+ if (nodeTypesConn2A.containsKey(depNodeType)) {
+ // This is the dependent node type that vertex A is using
+ depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
+ }
+ if (nodeTypesConn2B.containsKey(depNodeType)) {
+ // This is the dependent node type that vertex B is using
+ depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
+ }
+ }
+ if (depNodeVtxId4A.equals("")
+ || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
+ // Either they're not really dupes or there's some bad data - so
+ // don't pick either one
+ return nullVtx;
+ }
+ }
+
+ if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
+ // 2 - If they both have edges to all the same vertices,
+ // then return the one that can be reached uniquely via the
+ // key if that is the case or
+ // else the one with the lower vertexId
+
+ boolean allTheSame = true;
+ Iterator<String> iter = vtxIdsConn2A.iterator();
+ while (iter.hasNext()) {
+ String vtxIdConn2A = iter.next();
+ if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
+ allTheSame = false;
+ break;
+ }
+ }
+
+ if (allTheSame) {
+ // If everything is the same, but one of the two has a good
+ // pointer to it, then save that one. Otherwise, take the
+ // older one.
+ if( !onlyNodeThatIndexPointsToVidStr.equals("") ){
+ // only one is reachable via the index - choose that one.
+ if( onlyNodeThatIndexPointsToVidStr.equals(vidA.toString()) ){
+ preferredVtx = vtxA;
+ }
+ else if( onlyNodeThatIndexPointsToVidStr.equals(vidB.toString()) ){
+ preferredVtx = vtxB;
+ }
+ }
+ else if (vidA < vidB) {
+ preferredVtx = vtxA;
+ } else {
+ preferredVtx = vtxB;
+ }
+ }
+ } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
+ // 3 - VertexA is connected to more things than vtxB.
+ // We'll pick VtxA if its edges are a superset of vtxB's edges
+ // and it doesn't contradict the check for the index/key pointer.
+ boolean missingOne = false;
+ Iterator<String> iter = vtxIdsConn2B.iterator();
+ while (iter.hasNext()) {
+ String vtxIdConn2B = iter.next();
+ if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
+ missingOne = true;
+ break;
+ }
+ }
+ if (!missingOne) {
+ if( onlyNodeThatIndexPointsToVidStr.equals("")
+ || onlyNodeThatIndexPointsToVidStr.equals(vidA.toString()) ){
+ preferredVtx = vtxA;
+ }
+ }
+ } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
+ // 4 - VertexB is connected to more things than vtxA.
+ // We'll pick VtxB if its edges are a superset of vtxA's edges
+ // and it doesn't contradict the check for the index/key pointer.
+ boolean missingOne = false;
+ Iterator<String> iter = vtxIdsConn2A.iterator();
+ while (iter.hasNext()) {
+ String vtxIdConn2A = iter.next();
+ if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
+ missingOne = true;
+ break;
+ }
+ }
+ if (!missingOne) {
+ if( onlyNodeThatIndexPointsToVidStr.equals("")
+ || onlyNodeThatIndexPointsToVidStr.equals(vidB.toString()) ){
+ preferredVtx = vtxB;
+ }
+ }
+ } else {
+ preferredVtx = nullVtx;
+ }
+
+ return (preferredVtx);
+
+ } // end of pickOneOfTwoDupes()
+
+ /**
+ * Check and process dupes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param version the version
+ * @param nType the n type
+ * @param passedVertList the passed vert list
+ * @param dupeFixOn the dupe fix on
+ * @param deleteCandidateList the delete candidate list
+ * @param singleCommits the single commits
+ * @param alreadyFoundDupeGroups the already found dupe groups
+ * @return the array list
+ */
+ private List<String> checkAndProcessDupes(String transId,
+ String fromAppId, Graph g, GraphTraversalSource source, String version, String nType,
+ List<Vertex> passedVertList, Boolean dupeFixOn,
+ Set<String> deleteCandidateList, Boolean singleCommits,
+ ArrayList<String> alreadyFoundDupeGroups, Loader loader ) {
+
+ ArrayList<String> returnList = new ArrayList<>();
+ ArrayList<Vertex> checkVertList = new ArrayList<>();
+ ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<>();
+ Boolean noFilterList = true;
+ Iterator<String> afItr = alreadyFoundDupeGroups.iterator();
+ while (afItr.hasNext()) {
+ String dupeGrpStr = afItr.next();
+ String[] dupeArr = dupeGrpStr.split("\\|");
+ int lastIndex = dupeArr.length - 1;
+ for (int i = 0; i < lastIndex; i++) {
+ // Note: we don't want the last one...
+ String vidString = dupeArr[i];
+ alreadyFoundDupeVidArr.add(vidString);
+ noFilterList = false;
+ }
+ }
+
+ // For a given set of Nodes that were found with a set of KEY
+ // Parameters, (nodeType + key data) we will
+ // see if we find any duplicate nodes that need to be cleaned up. Note -
+ // it's legit to have more than one
+ // node with the same key data if the nodes depend on a parent for
+ // uniqueness -- as long as the two nodes
+ // don't hang off the same Parent.
+ // If we find duplicates, and we can figure out which of each set of
+ // duplicates is the one that we
+ // think should be preserved, we will record that. Whether we can tell
+ // which one should be
+ // preserved or not, we will return info about any sets of duplicates
+ // found.
+ //
+ // Each element in the returned arrayList might look like this:
+ // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
+ // couldn't figure out which one to keep)
+ // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
+ // thought the third one was the one that should survive)
+
+ // Because of the way the calling code loops over stuff, we can get the
+ // same data multiple times - so we should
+ // not process any vertices that we've already seen.
+
+ try {
+ Iterator<Vertex> pItr = passedVertList.iterator();
+ while (pItr.hasNext()) {
+ Vertex tvx = pItr.next();
+ String passedId = tvx.id().toString();
+ if (noFilterList || !alreadyFoundDupeVidArr.contains(passedId)) {
+ // We haven't seen this one before - so we should check it.
+ checkVertList.add(tvx);
+ }
+ }
+
+ if (checkVertList.size() < 2) {
+ // Nothing new to check.
+ return returnList;
+ }
+
+ if (loader.introspectorFromName(nType).isTopLevel()) {
+ // If this was a node that does NOT depend on other nodes for
+ // uniqueness, and we
+ // found more than one node using its key -- record the found
+ // vertices as duplicates.
+ String dupesStr = "";
+ for (int i = 0; i < checkVertList.size(); i++) {
+ dupesStr = dupesStr
+ + ((checkVertList.get(i))).id()
+ .toString() + "|";
+ }
+ if (dupesStr != "") {
+ Vertex prefV = getPreferredDupe(transId, fromAppId,
+ source, checkVertList, version, loader);
+ if (prefV == null) {
+ // We could not determine which duplicate to keep
+ dupesStr = dupesStr + "KeepVid=UNDETERMINED";
+ returnList.add(dupesStr);
+ } else {
+ dupesStr = dupesStr + "KeepVid=" + prefV.id();
+ Boolean didRemove = false;
+ if (dupeFixOn) {
+ didRemove = deleteNonKeepersIfAppropriate(g,
+ dupesStr, prefV.id().toString(),
+ deleteCandidateList, singleCommits);
+ }
+ if (didRemove) {
+ dupeGrpsDeleted++;
+ } else {
+ // keep them on our list
+ returnList.add(dupesStr);
+ }
+ }
+ }
+ } else {
+ // More than one node have the same key fields since they may
+ // depend on a parent node for uniqueness. Since we're finding
+ // more than one, we want to check to see if any of the
+ // vertices that have this set of keys (and are the same nodeType)
+ // are also pointing at the same 'parent' node.
+ // Note: for a given set of key data, it is possible that there
+ // could be more than one set of duplicates.
+ HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
+ transId, fromAppId, source, version, nType,
+ checkVertList, loader);
+ for (Map.Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash
+ .entrySet()) {
+ ArrayList<Vertex> thisParentsVertList = entry
+ .getValue();
+ if (thisParentsVertList.size() > 1) {
+ // More than one vertex found with the same key info
+ // hanging off the same parent/dependent node
+ String dupesStr = "";
+ for (int i = 0; i < thisParentsVertList.size(); i++) {
+ dupesStr = dupesStr
+ + ((thisParentsVertList
+ .get(i))).id() + "|";
+ }
+ if (dupesStr != "") {
+ Vertex prefV = getPreferredDupe(transId,
+ fromAppId, source, thisParentsVertList,
+ version, loader);
+
+ if (prefV == null) {
+ // We could not determine which duplicate to
+ // keep
+ dupesStr = dupesStr + "KeepVid=UNDETERMINED";
+ returnList.add(dupesStr);
+ } else {
+ Boolean didRemove = false;
+ dupesStr = dupesStr + "KeepVid="
+ + prefV.id().toString();
+ if (dupeFixOn) {
+ didRemove = deleteNonKeepersIfAppropriate(
+ g, dupesStr, prefV.id()
+ .toString(),
+ deleteCandidateList, singleCommits);
+ }
+ if (didRemove) {
+ dupeGrpsDeleted++;
+ } else {
+ // keep them on our list
+ returnList.add(dupesStr);
+ }
+ }
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. ", e);
+ }
+
+ return returnList;
+
+ }// End of checkAndProcessDupes()
+
+ /**
+ * Group verts by dep nodes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param version the version
+ * @param nType the n type
+ * @param passedVertList the passed vert list
+ * @return the hash map
+ * @throws AAIException the AAI exception
+ */
+ private HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes(
+ String transId, String fromAppId, GraphTraversalSource g, String version,
+ String nType, ArrayList<Vertex> passedVertList, Loader loader)
+ throws AAIException {
+ // Given a list of JanusGraph Vertices of one nodeType (see AAI-8956), group
+ // them together by the parent node they depend on.
+ // Ie. if given a list of ip address nodes (assumed to all have the
+ // same key info) they might sit under several different parent vertices.
+ // Under Normal conditions, there would only be one per parent -- but
+ // we're trying to find duplicates - so we
+ // allow for the case where more than one is under the same parent node.
+
+ HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>();
+ if (loader.introspectorFromName(nType).isTopLevel()) {
+ // This method really should not have been called if this is not the
+ // kind of node
+ // that depends on a parent for uniqueness, so just return the empty
+ // hash.
+ return retHash;
+ }
+
+ // Find out what types of nodes the passed in nodes can depend on
+ ArrayList<String> depNodeTypeL = new ArrayList<>();
+ Collection<String> depNTColl = loader.introspectorFromName(nType).getDependentOn();
+ Iterator<String> ntItr = depNTColl.iterator();
+ while (ntItr.hasNext()) {
+ depNodeTypeL.add(ntItr.next());
+ }
+ // For each vertex, we want find its depended-on/parent vertex so we
+ // can track what other vertexes that are dependent on that same guy.
+ if (passedVertList != null) {
+ Iterator<Vertex> iter = passedVertList.iterator();
+ while (iter.hasNext()) {
+ Vertex thisVert = iter.next();
+ Vertex tmpParentVtx = getConnectedParent( g, thisVert );
+ if( tmpParentVtx != null ) {
+ String parentNt = null;
+ Object obj = tmpParentVtx.<Object>property("aai-node-type").orElse(null);
+ if (obj != null) {
+ parentNt = obj.toString();
+ }
+ if (depNTColl.contains(parentNt)) {
+ // This must be the parent/dependent node
+ String parentVid = tmpParentVtx.id().toString();
+ if (retHash.containsKey(parentVid)) {
+ // add this vert to the list for this parent key
+ retHash.get(parentVid).add(thisVert);
+ } else {
+ // This is the first one we found on this parent
+ ArrayList<Vertex> vList = new ArrayList<>();
+ vList.add(thisVert);
+ retHash.put(parentVid, vList);
+ }
+ }
+ }
+ }
+ }
+
+ return retHash;
+
+ }// end of groupVertsByDepNodes()
+
+ /**
+ * Delete non keepers if appropriate.
+ *
+ * @param g the g
+ * @param dupeInfoString the dupe info string
+ * @param vidToKeep the vid to keep
+ * @param deleteCandidateList the delete candidate list
+ * @param singleCommits the single commits
+ * @return the boolean
+ */
+ private Boolean deleteNonKeepersIfAppropriate(Graph g,
+ String dupeInfoString, String vidToKeep,
+ Set<String> deleteCandidateList, Boolean singleCommits) {
+
+ Boolean deletedSomething = false;
+ // This assumes that the dupeInfoString is in the format of
+ // pipe-delimited vid's followed by
+ // ie. "3456|9880|keepVid=3456"
+ if (deleteCandidateList == null || deleteCandidateList.size() == 0) {
+ // No vid's on the candidate list -- so no deleting will happen on
+ // this run
+ return false;
+ }
+
+ String[] dupeArr = dupeInfoString.split("\\|");
+ ArrayList<String> idArr = new ArrayList<>();
+ int lastIndex = dupeArr.length - 1;
+ for (int i = 0; i <= lastIndex; i++) {
+ if (i < lastIndex) {
+ // This is not the last entry, it is one of the dupes,
+ String vidString = dupeArr[i];
+ idArr.add(vidString);
+ } else {
+ // This is the last entry which should tell us if we have a
+ // preferred keeper
+ String prefString = dupeArr[i];
+ if (prefString.equals("KeepVid=UNDETERMINED")) {
+ // They sent us a bad string -- nothing should be deleted if
+ // no dupe could be tagged as preferred
+ return false;
+ } else {
+ // If we know which to keep, then the prefString should look
+ // like, "KeepVid=12345"
+ String[] prefArr = prefString.split("=");
+ if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("Bad format. Expecting KeepVid=999999");
+ return false;
+ } else {
+ String keepVidStr = prefArr[1];
+ if (idArr.contains(keepVidStr)) {
+ idArr.remove(keepVidStr);
+
+ // So now, the idArr should just contain the vid's
+ // that we want to remove.
+ for (int x = 0; x < idArr.size(); x++) {
+ boolean okFlag = true;
+ String thisVid = idArr.get(x);
+ if (deleteCandidateList.contains(thisVid)) {
+ // This vid is a valid delete candidate from
+ // a prev. run, so we can remove it.
+ try {
+ long longVertId = Long
+ .parseLong(thisVid);
+ Vertex vtx = g
+ .traversal().V(longVertId).next();
+ vtx.remove();
+
+ if (singleCommits) {
+ // NOTE - the singleCommits option is not used in normal processing
+ g.tx().commit();
+ g = AAIGraph.getInstance().getGraph().newTransaction();
+ }
+ } catch (Exception e) {
+ okFlag = false;
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("ERROR trying to delete VID = " + thisVid + " " + LogFormatTools.getStackTop(e));
+ }
+ if (okFlag) {
+ LOGGER.info(" DELETED VID = " + thisVid);
+ deletedSomething = true;
+ }
+ }
+ }
+ } else {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error("ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
+ + dupeInfoString + "]");
+ return false;
+ }
+ }
+ }// else we know which one to keep
+ }// else last entry
+ }// for each vertex in a group
+
+ return deletedSomething;
+
+ }// end of deleteNonKeepersIfAppropriate()
+
+
+ /**
+ * Gets the node just using key params.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param graph the graph
+ * @param nodeType the node type
+ * @param keyPropsHash the key props hash
+ * @param apiVersion the api version
+ * @return the node just using key params
+ * @throws AAIException the AAI exception
+ */
+ public List <Vertex> getNodeJustUsingKeyParams( String transId, String fromAppId, GraphTraversalSource graph, String nodeType,
+ HashMap<String,Object> keyPropsHash, String apiVersion ) throws AAIException{
+
+ List <Vertex> retVertList = new ArrayList <> ();
+
+ // We assume that all NodeTypes have at least one key-property defined.
+ // Note - instead of key-properties (the primary key properties), a user could pass
+ // alternate-key values if they are defined for the nodeType.
+ List<String> kName = new ArrayList<>();
+ List<Object> kVal = new ArrayList<>();
+ if( keyPropsHash == null || keyPropsHash.isEmpty() ) {
+ throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request. NodeType = [" + nodeType + "]. ");
+ }
+
+ int i = -1;
+ for( Map.Entry<String, Object> entry : keyPropsHash.entrySet() ){
+ i++;
+ kName.add(i, entry.getKey());
+ kVal.add(i, entry.getValue());
+ }
+ int topPropIndex = i;
+ Vertex tiV = null;
+ String propsAndValuesForMsg = "";
+ Iterator <Vertex> verts = null;
+
+ try {
+ if( topPropIndex == 0 ){
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
+ verts= graph.V().has(kName.get(0),kVal.get(0)).has("aai-node-type",nodeType);
+ }
+ else if( topPropIndex == 1 ){
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
+ + kName.get(1) + " = " + kVal.get(1) + ") ";
+ verts = graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has("aai-node-type",nodeType);
+ }
+ else if( topPropIndex == 2 ){
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
+ + kName.get(1) + " = " + kVal.get(1) + ", "
+ + kName.get(2) + " = " + kVal.get(2) + ") ";
+ verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has("aai-node-type",nodeType);
+ }
+ else if( topPropIndex == 3 ){
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
+ + kName.get(1) + " = " + kVal.get(1) + ", "
+ + kName.get(2) + " = " + kVal.get(2) + ", "
+ + kName.get(3) + " = " + kVal.get(3) + ") ";
+ verts= graph.V().has(kName.get(0),kVal.get(0)).has(kName.get(1),kVal.get(1)).has(kName.get(2),kVal.get(2)).has(kName.get(3),kVal.get(3)).has("aai-node-type",nodeType);
+ }
+ else {
+ throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
+ }
+ }
+ catch( Exception ex ){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ LOGGER.error( " ERROR trying to get node for: [" + propsAndValuesForMsg + "]" + LogFormatTools.getStackTop(ex));
+ }
+
+ if( verts != null ){
+ while( verts.hasNext() ){
+ tiV = verts.next();
+ retVertList.add(tiV);
+ }
+ }
+
+ if( retVertList.size() == 0 ){
+ LOGGER.debug("DEBUG No node found for nodeType = [" + nodeType +
+ "], propsAndVal = " + propsAndValuesForMsg );
+ }
+
+ return retVertList;
+
+ }// End of getNodeJustUsingKeyParams()
+
+ /**
+ * Show all edges for node.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param tVert the t vert
+ * @return the array list
+ */
+ private ArrayList <String> showAllEdgesForNode( String transId, String fromAppId, Vertex tVert ){
+
+ ArrayList <String> retArr = new ArrayList <> ();
+ Iterator <Edge> eI = tVert.edges(Direction.IN);
+ if( ! eI.hasNext() ){
+ retArr.add("No IN edges were found for this vertex. ");
+ }
+ while( eI.hasNext() ){
+ Edge ed = eI.next();
+ String lab = ed.label();
+ Vertex vtx;
+ if (tVert.equals(ed.inVertex())) {
+ vtx = ed.outVertex();
+ } else {
+ vtx = ed.inVertex();
+ }
+ if( vtx == null ){
+ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ }
+ else {
+ String nType = vtx.<String>property("aai-node-type").orElse(null);
+ String vid = vtx.id().toString();
+ retArr.add("Found an IN edge (" + lab + ") to this vertex from a [" + nType + "] node with VtxId = " + vid );
+
+ }
+ }
+
+ eI = tVert.edges(Direction.OUT);
+ if( ! eI.hasNext() ){
+ retArr.add("No OUT edges were found for this vertex. ");
+ }
+ while( eI.hasNext() ){
+ Edge ed = eI.next();
+ String lab = ed.label();
+ Vertex vtx;
+ if (tVert.equals(ed.inVertex())) {
+ vtx = ed.outVertex();
+ } else {
+ vtx = ed.inVertex();
+ }
+ if( vtx == null ){
+ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ }
+ else {
+ String nType = vtx.<String>property("aai-node-type").orElse(null);
+ String vid = vtx.id().toString();
+ retArr.add("Found an OUT edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid );
+ }
+ }
+ return retArr;
+ }
+
+
+ /**
+ * Show properties for node.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param tVert the t vert
+ * @return the array list
+ */
+ private ArrayList <String> showPropertiesForNode( String transId, String fromAppId, Vertex tVert ){
+
+ ArrayList <String> retArr = new ArrayList <> ();
+ if( tVert == null ){
+ retArr.add("null Node object passed to showPropertiesForNode()\n");
+ }
+ else {
+ String nodeType = "";
+ Object ob = tVert.<Object>property("aai-node-type").orElse(null);
+ if( ob == null ){
+ nodeType = "null";
+ }
+ else{
+ nodeType = ob.toString();
+ }
+
+ retArr.add(" AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
+ retArr.add(" Property Detail: ");
+ Iterator<VertexProperty<Object>> pI = tVert.properties();
+ while( pI.hasNext() ){
+ VertexProperty<Object> tp = pI.next();
+ Object val = tp.value();
+ retArr.add("Prop: [" + tp.key() + "], val = [" + val + "] ");
+ }
+ }
+ return retArr;
+ }
+
+
+ private ArrayList <Vertex> getConnectedNodes(GraphTraversalSource g, Vertex startVtx )
+ throws AAIException {
+
+ ArrayList <Vertex> retArr = new ArrayList <> ();
+ if( startVtx == null ){
+ return retArr;
+ }
+ else {
+ GraphTraversal<Vertex, Vertex> modPipe = null;
+ modPipe = g.V(startVtx).both();
+ if( modPipe != null && modPipe.hasNext() ){
+ while( modPipe.hasNext() ){
+ Vertex conVert = modPipe.next();
+ retArr.add(conVert);
+ }
+ }
+ }
+ return retArr;
+
+ }// End of getConnectedNodes()
+
+
+ private ArrayList <Vertex> getConnectedChildrenOfOneType( GraphTraversalSource g,
+ Vertex startVtx, String childNType ) throws AAIException{
+
+ ArrayList <Vertex> childList = new ArrayList <> ();
+ Iterator <Vertex> vertI = g.V(startVtx).union(__.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).inV(), __.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).outV());
+
+ Vertex tmpVtx = null;
+ while( vertI != null && vertI.hasNext() ){
+ tmpVtx = vertI.next();
+ Object ob = tmpVtx.<Object>property("aai-node-type").orElse(null);
+ if (ob != null) {
+ String tmpNt = ob.toString();
+ if( tmpNt.equals(childNType)){
+ childList.add(tmpVtx);
+ }
+ }
+ }
+
+ return childList;
+
+ }// End of getConnectedChildrenOfOneType()
+
+
+ private Vertex getConnectedParent( GraphTraversalSource g,
+ Vertex startVtx ) throws AAIException{
+
+ Vertex parentVtx = null;
+ Iterator <Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
+
+ while( vertI != null && vertI.hasNext() ){
+ // Note - there better only be one!
+ parentVtx = vertI.next();
+ }
+
+ return parentVtx;
+
+ }// End of getConnectedParent()
+
+
+ private long figureWindowStartTime( int timeWindowMinutes ){
+ // Given a window size, calculate what the start-timestamp would be.
+
+ if( timeWindowMinutes <= 0 ){
+ // This just means that there is no window...
+ return 0;
+ }
+ long unixTimeNow = System.currentTimeMillis();
+ long windowInMillis = timeWindowMinutes * 60L * 1000;
+
+ long startTimeStamp = unixTimeNow - windowInMillis;
+
+ return startTimeStamp;
+ } // End of figureWindowStartTime()
+
+
+ /**
+ * Collect Duplicate Sets for nodes that are NOT dependent on parent nodes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param version the version
+ * @param nType the n type
+ * @param passedVertList the passed vert list
+ * @return the array list
+ */
+ private ArrayList<ArrayList<Vertex>> getDupeSets4NonDepNodes( String transId,
+ String fromAppId, Graph g, String version, String nType,
+ ArrayList<Vertex> passedVertList,
+ ArrayList <String> keyPropNamesArr,
+ Loader loader ) {
+
+ ArrayList<ArrayList<Vertex>> returnList = new ArrayList<ArrayList<Vertex>>();
+
+ // We've been passed a set of nodes that we want to check.
+ // They are all NON-DEPENDENT nodes of the same nodeType meaning that they should be
+ // unique in the DB based on their KEY DATA alone. So, if
+ // we group them by their key data - if any key has more than one
+ // vertex mapped to it, those vertices are dupes.
+ //
+ // When we find duplicates, we group them in an ArrayList (there can be
+ // more than one duplicate for one set of key data)
+ // Then these dupeSets are grouped up and returned.
+ //
+
+ HashMap <String, ArrayList<String>> keyVals2VidHash = new HashMap <String, ArrayList<String>>();
+ HashMap <String,Vertex> vtxHash = new HashMap <String,Vertex>();
+ Iterator<Vertex> pItr = passedVertList.iterator();
+ while (pItr.hasNext()) {
+ try {
+ Vertex tvx = pItr.next();
+ String thisVid = tvx.id().toString();
+ vtxHash.put(thisVid, tvx);
+
+ // if there are more than one vertexId mapping to the same keyProps -- they are dupes
+ // we dont check till later since a set can contain more than 2.
+ String hKey = getNodeKeyValString( tvx, keyPropNamesArr );
+ if( hKey.equals("") ){
+ // When we have corrupted data, hKey comes back as an empty string
+ // We will just skip this entry since it is not a Dupe - it is
+ // corrupted data which should be picked up in other checks.
+ continue;
+ }
+ if( keyVals2VidHash.containsKey(hKey) ){
+ // We've already seen this key
+ ArrayList <String> tmpVL = (ArrayList <String>)keyVals2VidHash.get(hKey);
+ tmpVL.add(thisVid);
+ keyVals2VidHash.put(hKey, tmpVL);
+ }
+ else {
+ // First time for this key
+ ArrayList <String> tmpVL = new ArrayList <String>();
+ tmpVL.add(thisVid);
+ keyVals2VidHash.put(hKey, tmpVL);
+ }
+ }
+ catch (Exception e) {
+ LOGGER.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. ", e);
+ }
+ }
+
+ for( Map.Entry<String, ArrayList<String>> entry : keyVals2VidHash.entrySet() ){
+ ArrayList <String> vidList = entry.getValue();
+ try {
+ if( !vidList.isEmpty() && vidList.size() > 1 ){
+ // There are more than one vertex id's using the same key info
+ ArrayList <Vertex> vertList = new ArrayList <Vertex> ();
+ for (int i = 0; i < vidList.size(); i++) {
+ String tmpVid = vidList.get(i);
+ vertList.add(vtxHash.get(tmpVid));
+ }
+ returnList.add(vertList);
+ }
+ }
+ catch (Exception e) {
+ LOGGER.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. ", e);
+ }
+
+ }
+ return returnList;
+
+ }// End of getDupeSets4NonDepNodes()
+
+
+ /**
+ * Get values of the key properties for a node as a single string
+ *
+ * @param tvx the vertex to pull the properties from
+ * @param keyPropNamesArr collection of key prop names
+ * @return a String of concatenated values
+ */
+ private String getNodeKeyValString( Vertex tvx,
+ ArrayList <String> keyPropNamesArr ) {
+
+ String retString = "";
+ Iterator <String> propItr = keyPropNamesArr.iterator();
+ while( propItr.hasNext() ){
+ String propName = propItr.next();
+ if( tvx != null ){
+ Object propValObj = tvx.property(propName).orElse(null);
+ if( propValObj == null ){
+ LOGGER.warn(" >>> WARNING >>> could not find this key-property for this vertex. propName = ["
+ + propName + "], VID = " + tvx.id().toString() );
+ }
+ else {
+ retString = " " + retString + propValObj.toString();
+ }
+ }
+ }
+ return retString;
+
+ }// End of getNodeKeyValString()
+
+
+ private String findJustOneUsingIndex( String transId, String fromAppId,
+ GraphTraversalSource gts, HashMap <String,Object> keyPropValsHash,
+ String nType, Long vidAL, Long vidBL, String apiVer){
+
+ // See if querying by JUST the key params (which should be indexed) brings back
+ // ONLY one of the two vertices. Ie. the db still has a pointer to one of them
+ // and the other one is sort of stranded.
+ String returnVid = "";
+
+ try {
+ List <Vertex> tmpVertList = getNodeJustUsingKeyParams( transId, fromAppId, gts,
+ nType, keyPropValsHash, apiVer );
+ if( tmpVertList != null && tmpVertList.size() == 1 ){
+ // We got just one - if it matches one of the ones we're looking
+ // for, then return that VID
+ Vertex tmpV = tmpVertList.get(0);
+ String thisVid = tmpV.id().toString();
+ if( thisVid.equals(vidAL.toString()) || thisVid.equals(vidBL.toString()) ){
+ String msg = " vid = " + thisVid + " is one of two that the DB can retrieve directly ------";
+ //System.out.println(msg);
+ LOGGER.info(msg);
+ returnVid = thisVid;
+ }
+ }
+ }
+ catch ( AAIException ae ){
+ String emsg = "Error trying to get node just by key " + ae.getMessage();
+ //System.out.println(emsg);
+ LOGGER.error(emsg);
+ }
+
+ return returnVid;
+
+ }// End of findJustOneUsingIndex()
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java b/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java
new file mode 100644
index 0000000..85a127f
--- /dev/null
+++ b/src/main/java/org/onap/aai/datagrooming/DataGroomingTasks.java
@@ -0,0 +1,204 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datagrooming;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConfig;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+@Component
+@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
+public class DataGroomingTasks {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataGroomingTasks.class);
+ private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
+ @Autowired
+ private LoaderFactory loaderFactory;
+
+ @Autowired
+ private SchemaVersions schemaVersions;
+
+ @Scheduled(cron = "${datagroomingtasks.cron}" )
+ public void groomingScheduleTask() throws AAIException, Exception {
+
+ LoggingContext.init();
+ LoggingContext.requestId(UUID.randomUUID().toString());
+ LoggingContext.partnerName("AAI");
+ LoggingContext.targetEntity("CronApp");
+ LoggingContext.component("dataGrooming");
+ LoggingContext.serviceName("groomingScheduleTask");
+ LoggingContext.targetServiceName("groomingScheduleTask");
+ LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
+
+
+ if(!"true".equals(AAIConfig.get("aai.disable.check.grooming.running", "false"))){
+ if(checkIfDataGroomingIsRunning()){
+ LOGGER.info("Data Grooming is already running on the system");
+ return;
+ }
+ }
+
+ LOGGER.info("Started cron job dataGrooming @ " + dateFormat.format(new Date()));
+
+ Map<String, String> dataGroomingFlagMap = new HashMap<>();
+ append("enableautofix" , AAIConfig.get("aai.datagrooming.enableautofix"), dataGroomingFlagMap);
+ append("enabledupefixon" , AAIConfig.get("aai.datagrooming.enabledupefixon"), dataGroomingFlagMap);
+ append("enabledontfixorphans" , AAIConfig.get("aai.datagrooming.enabledontfixorphans"), dataGroomingFlagMap);
+ append("enabletimewindowminutes" , AAIConfig.get("aai.datagrooming.enabletimewindowminutes"), dataGroomingFlagMap);
+ append("enableskiphostcheck" , AAIConfig.get("aai.datagrooming.enableskiphostcheck"), dataGroomingFlagMap);
+ append("enablesleepminutes" , AAIConfig.get("aai.datagrooming.enablesleepminutes"), dataGroomingFlagMap);
+ append("enableedgesonly" , AAIConfig.get("aai.datagrooming.enableedgesonly"), dataGroomingFlagMap);
+ append("enableskipedgechecks" , AAIConfig.get("aai.datagrooming.enableskipedgechecks"), dataGroomingFlagMap);
+ append("enablemaxfix" , AAIConfig.get("aai.datagrooming.enablemaxfix"), dataGroomingFlagMap);
+ append("enablesinglecommits" , AAIConfig.get("aai.datagrooming.enablesinglecommits"), dataGroomingFlagMap);
+ append("enabledupecheckoff" , AAIConfig.get("aai.datagrooming.enabledupecheckoff"), dataGroomingFlagMap);
+ append("enableghost2checkoff" , AAIConfig.get("aai.datagrooming.enableghost2checkoff"), dataGroomingFlagMap);
+ append("enableghost2fixon" , AAIConfig.get("aai.datagrooming.enableghost2fixon"), dataGroomingFlagMap);
+ append("enablef" , AAIConfig.get("aai.datagrooming.enablef"), dataGroomingFlagMap);
+ append("fvalue" , AAIConfig.get("aai.datagrooming.fvalue"), dataGroomingFlagMap);
+ append("timewindowminutesvalue" , AAIConfig.get("aai.datagrooming.timewindowminutesvalue"), dataGroomingFlagMap);
+ append("sleepminutesvalue" , AAIConfig.get("aai.datagrooming.sleepminutesvalue"), dataGroomingFlagMap);
+ append("maxfixvalue" , AAIConfig.get("aai.datagrooming.maxfixvalue"), dataGroomingFlagMap);
+
+ if(LOGGER.isDebugEnabled()){
+ LOGGER.debug("DataGrooming Flag Values : ");
+ dataGroomingFlagMap.forEach((key, val) -> LOGGER.debug("Key: {} Value: {}", key, val));
+ }
+
+ List<String> paramsArray = new ArrayList();
+ try {
+ if("true".equals(dataGroomingFlagMap.get("enableautofix"))){
+ paramsArray.add("-autoFix");
+ }
+ if("true".equals(dataGroomingFlagMap.get("enabledupefixon"))){
+ paramsArray.add("-dupeFixOn");
+ }
+ if("true".equals(dataGroomingFlagMap.get("enabledontfixorphans"))){
+ paramsArray.add("-dontFixOrphans");
+ }
+ if("true".equals(dataGroomingFlagMap.get("enabletimewindowminutes"))){
+ paramsArray.add("-timeWindowMinutes");
+ paramsArray.add(dataGroomingFlagMap.get("enabletimewindowminutesvalue"));
+ }
+ if("true".equals(dataGroomingFlagMap.get("enableskiphostcheck"))){
+ paramsArray.add("-skipHostCheck");
+ }
+
+ if("true".equals(dataGroomingFlagMap.get("enablesleepminutes"))) {
+ paramsArray.add("-sleepMinutes");
+ paramsArray.add(dataGroomingFlagMap.get("sleepminutesvalue"));
+ }
+
+ if("true".equals(dataGroomingFlagMap.get("enableedgesonly"))){
+ paramsArray.add("-edgesOnly");
+ }
+ if("true".equals(dataGroomingFlagMap.get("enableskipedgechecks"))) {
+ paramsArray.add("-skipEdgeChecks");
+ }
+
+ if("true".equals(dataGroomingFlagMap.get("enablemaxfix"))) {
+ paramsArray.add("-maxFix");
+ paramsArray.add(dataGroomingFlagMap.get("maxfixvalue"));
+ }
+ if("true".equals(dataGroomingFlagMap.get("enablesinglecommits"))){
+ paramsArray.add("-singleCommits");
+ }
+ if("true".equals(dataGroomingFlagMap.get("enabledupecheckoff"))){
+ paramsArray.add("-dupeCheckOff");
+ }
+ if("true".equals(dataGroomingFlagMap.get("enableghost2checkoff"))){
+ paramsArray.add("-ghost2CheckOff");
+ }
+ if("true".equals(dataGroomingFlagMap.get("enableghost2fixon"))){
+ paramsArray.add("-ghost2FixOn");
+ }
+
+ if("true".equals(dataGroomingFlagMap.get("enablef"))) {
+ paramsArray.add("-f");
+ paramsArray.add(dataGroomingFlagMap.get("fvalue"));
+ }
+
+ DataGrooming dataGrooming = new DataGrooming(loaderFactory, schemaVersions);
+ String[] paramsList = paramsArray.toArray(new String[0]);
+ if (AAIConfig.get("aai.cron.enable.dataGrooming").equals("true")) {
+ dataGrooming.execute(paramsList);
+ System.out.println("returned from main method ");
+ }
+ }
+ catch (Exception e) {
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for dataGrooming"+e.toString());
+ LOGGER.info("AAI_4000", "Exception running cron job for dataGrooming"+e.toString());
+ throw e;
+ } finally {
+ LOGGER.info("Ended cron job dataGrooming @ " + dateFormat.format(new Date()));
+ LoggingContext.clear();
+ }
+ }
+
+ private boolean checkIfDataGroomingIsRunning(){
+
+ Process process = null;
+
+ int count = 0;
+ try {
+ process = new ProcessBuilder().command("bash", "-c", "ps -ef | grep '[D]ataGrooming'").start();
+ InputStream is = process.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ while (br.readLine() != null){
+ count++;
+ }
+
+ int exitVal = process.waitFor();
+ LOGGER.info("Exit value of the dataGrooming check process: " + exitVal);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ if(count > 0){
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private void append(String key, String value, Map<String, String> hashMap){
+ hashMap.put(key, value);
+ }
+}
diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java
new file mode 100644
index 0000000..12815ee
--- /dev/null
+++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshot.java
@@ -0,0 +1,835 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ *
+ * ECOMP is a trademark and service mark of AT&T Intellectual Property.
+ */
+package org.onap.aai.datasnapshot;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.SequenceInputStream;
+import java.util.*;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import org.apache.commons.configuration.PropertiesConfiguration;
+
+import org.apache.tinkerpop.gremlin.structure.io.IoCore;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.LegacyGraphSONReader;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.AAISystemExitUtil;
+import org.onap.aai.util.FormatDate;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.util.JanusGraphCleanup;
+
+public class DataSnapshot {
+
+ private static EELFLogger LOGGER;
+
+ /* Using realtime d */
+ private static final String REALTIME_DB = "realtime";
+
+ private static final Set<String> SNAPSHOT_RELOAD_COMMANDS = new HashSet<>();
+
+ static {
+ SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_LEGACY_DATA");
+ SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA");
+ SNAPSHOT_RELOAD_COMMANDS.add("RELOAD_DATA_MULTI");
+ }
+
+
+ /**
+ * The main method.
+ *
+ * @param args
+ * the arguments
+ */
+ public static void main(String[] args) {
+
+ boolean success = true;
+
+ // Set the logging file properties to be used by EELFManager
+ System.setProperty("aai.service.name", DataSnapshot.class.getSimpleName());
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ LOGGER = EELFManager.getInstance().getLogger(DataSnapshot.class);
+ Boolean dbClearFlag = false;
+ JanusGraph graph = null;
+ String command = "JUST_TAKE_SNAPSHOT"; // This is the default
+ String oldSnapshotFileName = "";
+
+ Long vertAddDelayMs = 1L; // Default value
+ Long edgeAddDelayMs = 1L; // Default value
+
+ Long failureDelayMs = 50L; // Default value
+ Long retryDelayMs = 1500L; // Default value
+ int maxErrorsPerThread = 25; // Default value
+ Long vertToEdgeProcDelay = 9000L; // Default value
+ Long staggerThreadDelay = 5000L; // Default value
+
+ int threadCount = 0;
+ Boolean debugFlag = false;
+ int debugAddDelayTime = 1; // Default to 1 millisecond
+
+ boolean isExistingTitan = false;
+
+ if (args.length >= 1) {
+ command = args[0];
+ }
+
+ if( SNAPSHOT_RELOAD_COMMANDS.contains(command)){
+ if (args.length == 2) {
+ // If re-loading, they need to also pass the snapshot file name to use.
+ // We expected the file to be found in our snapshot directory.
+ oldSnapshotFileName = args[1];
+ }
+ }
+ else if( command.equals("THREADED_SNAPSHOT") ){
+ if (args.length == 2) {
+ // If doing a "threaded" snapshot, they need to specify how many threads to use
+ try {
+ threadCount = Integer.parseInt(args[1]);
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) threadCount passed to DataSnapshot [" + args[1] + "]");
+ LOGGER.debug("Bad (non-integer) threadCount passed to DataSnapshot [" + args[1] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( threadCount < 1 || threadCount > 100 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (1-100) threadCount passed to DataSnapshot [" + args[1] + "]");
+ LOGGER.debug("Out of range (1-100) threadCount passed to DataSnapshot [" + args[1] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount );
+ }
+ else if (args.length == 3) {
+ // If doing a "threaded" snapshot, they need to specify how many threads to use
+ // They can also use debug mode if they pass the word "DEBUG" to do the nodes one at a time to see where it breaks.
+ try {
+ threadCount = Integer.parseInt(args[1]);
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) threadCount passed to DataSnapshot [" + args[1] + "]");
+ LOGGER.debug("Bad (non-integer) threadCount passed to DataSnapshot [" + args[1] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( threadCount < 1 || threadCount > 100 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (1-100) threadCount passed to DataSnapshot [" + args[1] + "]");
+ LOGGER.debug("Out of range (1-100) threadCount passed to DataSnapshot [" + args[1] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( args[2].equals("DEBUG") ){
+ debugFlag = true;
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount +
+ ", and DEBUG mode set ON. ");
+ }
+ else if (args.length == 4) {
+ // If doing a "threaded" snapshot, they need to specify how many threads to use (param 1)
+ // They can also use debug mode if they pass the word "DEBUG" to do the nodes one (param 2)
+ // They can also pass a delayTimer - how many milliseconds to put between each node's ADD (param 3)
+ try {
+ threadCount = Integer.parseInt(args[1]);
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) threadCount passed to DataSnapshot [" + args[1] + "]");
+ LOGGER.debug("Bad (non-integer) threadCount passed to DataSnapshot [" + args[1] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( threadCount < 1 || threadCount > 100 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (1-100) threadCount passed to DataSnapshot [" + args[1] + "]");
+ LOGGER.debug("Out of range (1-100) threadCount passed to DataSnapshot [" + args[1] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( args[2].equals("DEBUG") ){
+ debugFlag = true;
+ }
+ try {
+ debugAddDelayTime = Integer.parseInt(args[3]);
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) debugAddDelayTime passed to DataSnapshot [" + args[3] + "]");
+ LOGGER.debug("Bad (non-integer) debugAddDelayTime passed to DataSnapshot [" + args[3] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ LOGGER.debug(" Will do Threaded Snapshot with threadCount = " + threadCount +
+ ", DEBUG mode ON and addDelayTimer = " + debugAddDelayTime + " mSec. ");
+ }
+ else {
+ ErrorLogHelper.logError("AAI_6128", "Wrong param count (should be 2,3 or 4) when using THREADED_SNAPSHOT.");
+ LOGGER.debug("Wrong param count (should be 2,3 or 4) when using THREADED_SNAPSHOT.");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ }
+ else if( command.equals("MULTITHREAD_RELOAD") ){
+ // Note - this will use as many threads as the snapshot file is
+ // broken up into. (up to a limit)
+ if (args.length == 2) {
+ // Since they are re-loading, they need to pass the snapshot file name to use.
+ // We expected the file to be found in our snapshot directory. Note - if
+ // it is a multi-part snapshot, then this should be the root of the name.
+ // We will be using the default delay timers.
+ oldSnapshotFileName = args[1];
+ }
+ else if (args.length == 7) {
+ // Since they are re-loading, they need to pass the snapshot file name to use.
+ // We expected the file to be found in our snapshot directory. Note - if
+ // it is a multi-part snapshot, then this should be the root of the name.
+ oldSnapshotFileName = args[1];
+ // They should be passing the timers in in this order:
+ // vertDelay, edgeDelay, failureDelay, retryDelay
+ vertAddDelayMs = Long.parseLong(args[2]);
+ edgeAddDelayMs = Long.parseLong(args[3]);
+ failureDelayMs = Long.parseLong(args[4]);
+ retryDelayMs = Long.parseLong(args[5]);
+ try {
+ maxErrorsPerThread = Integer.parseInt(args[6]);
+ }
+ catch ( NumberFormatException nfe ){
+ ErrorLogHelper.logError("AAI_6128", "Bad (non-integer) maxErrorsPerThread passed to DataSnapshot [" + args[6] + "]");
+ LOGGER.debug("Bad (non-integer) maxErrorsPerThread passed to DataSnapshot [" + args[6] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ if( maxErrorsPerThread < 1 ){
+ ErrorLogHelper.logError("AAI_6128", "Out of range (>0) maxErrorsPerThread passed to DataSnapshot [" + args[6] + "]");
+ LOGGER.debug("Out of range (>0) maxErrorsPerThread passed to DataSnapshot [" + args[6] + "]");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ }
+ else {
+ ErrorLogHelper.logError("AAI_6128", "Wrong param count (should be either 2 or 7) when using MUTLITHREAD_RELOAD.");
+ LOGGER.debug("Wrong param count (should be 2 or 7) when using MUTLITHREAD_RELOAD.");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ }
+ else if (command.equals("CLEAR_ENTIRE_DATABASE")) {
+ if (args.length >= 2) {
+ oldSnapshotFileName = args[1];
+ }
+ if (args.length == 3) {
+ String titanFlag = args[2];
+ if ("titan".equalsIgnoreCase(titanFlag)) {
+ isExistingTitan = true;
+ }
+ }
+ }
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+
+ AAIConfig.init();
+ ErrorLogHelper.loadProperties();
+ LOGGER.debug("Command = " + command + ", oldSnapshotFileName = [" + oldSnapshotFileName + "].");
+ String targetDir = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "dataSnapshots";
+
+ // Make sure the dataSnapshots directory is there
+ new File(targetDir).mkdirs();
+
+ LOGGER.debug(" ---- NOTE --- about to open graph (takes a little while) ");
+
+ if (command.equals("JUST_TAKE_SNAPSHOT")) {
+ // ------------------------------------------
+ // They just want to take a snapshot.
+ // ------------------------------------------
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dteStr = fd.getDateTime();
+ String newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr;
+ graph = AAIGraph.getInstance().getGraph();
+
+ graph.io(IoCore.graphson()).writeGraph(newSnapshotOutFname);
+
+ LOGGER.debug("Snapshot written to " + newSnapshotOutFname);
+
+ }
+ else if (command.equals("THREADED_SNAPSHOT")) {
+ // ---------------------------------------------------------------------
+ // They want the creation of the snapshot to be spread out via threads
+ // ---------------------------------------------------------------------
+
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dteStr = fd.getDateTime();
+ String newSnapshotOutFname = targetDir + AAIConstants.AAI_FILESEP + "dataSnapshot.graphSON." + dteStr;
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ graph = AAIGraph.getInstance().getGraph();
+ LOGGER.debug(" Successfully got the Graph instance. ");
+ long timeA = System.nanoTime();
+
+ LOGGER.debug(" Need to divide vertexIds across this many threads: " + threadCount );
+ HashMap <String,ArrayList> vertListHash = new HashMap <String,ArrayList> ();
+ for( int t = 0; t < threadCount; t++ ){
+ ArrayList <Vertex> vList = new ArrayList <Vertex> ();
+ String tk = "" + t;
+ vertListHash.put( tk, vList);
+ }
+ LOGGER.debug("Count how many nodes are in the db. ");
+ long totalVertCount = graph.traversal().V().count().next();
+ LOGGER.debug(" Total Count of Nodes in DB = " + totalVertCount + ".");
+ long nodesPerFile = totalVertCount / threadCount;
+ LOGGER.debug(" Thread count = " + threadCount + ", each file will get (roughly): " + nodesPerFile + " nodes.");
+ long timeA2 = System.nanoTime();
+ long diffTime = timeA2 - timeA;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To count all vertices in DB it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ long vtxIndex = 0;
+ int currentTNum = 0;
+ String currentTKey = "0";
+ long thisThrIndex = 0;
+ Iterator <Vertex> vtxItr = graph.vertices();
+ while( vtxItr.hasNext() ){
+ // Divide up all the vertices so we can process them on different threads
+ vtxIndex++;
+ thisThrIndex++;
+ if( (thisThrIndex > nodesPerFile) && (currentTNum < threadCount -1) ){
+ // We will need to start adding to the Hash for the next thread
+ currentTNum++;
+ currentTKey = "" + currentTNum;
+ thisThrIndex = 0;
+ }
+ (vertListHash.get(currentTKey)).add(vtxItr.next());
+ }
+
+ long timeB = System.nanoTime();
+ diffTime = timeB - timeA2;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To Loop over all vertices, and put them into sub-Arrays it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ // Need to print out each set of vertices using it's own thread
+ ArrayList <Thread> threadArr = new ArrayList <Thread> ();
+ for( int thNum = 0; thNum < threadCount; thNum++ ){
+ String thNumStr = "" + thNum;
+ String subFName = newSnapshotOutFname + ".P" + thNumStr;
+ Thread thr = new Thread(new PrintVertexDetails(graph, subFName, vertListHash.get(thNumStr),
+ debugFlag, debugAddDelayTime) );
+ thr.start();
+ threadArr.add(thr);
+ }
+
+ // Make sure all the threads finish before moving on.
+ for( int thNum = 0; thNum < threadCount; thNum++ ){
+ if( null != threadArr.get(thNum) ){
+ (threadArr.get(thNum)).join();
+ }
+ }
+
+ long timeC = System.nanoTime();
+ diffTime = timeC - timeB;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To write all the data out to snapshot files, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+
+ } else if( command.equals("MULTITHREAD_RELOAD") ){
+ // ---------------------------------------------------------------------
+ // They want the RELOAD of the snapshot to be spread out via threads
+ // NOTE - it will only use as many threads as the number of files the
+ // snapshot is written to. Ie. if you have a single-file snapshot,
+ // then this will be single-threaded.
+ //
+ ArrayList <File> snapFilesArr = getFilesToProcess(targetDir, oldSnapshotFileName, false);
+ int fCount = snapFilesArr.size();
+ Iterator <File> fItr = snapFilesArr.iterator();
+
+ JanusGraph graph1 = AAIGraph.getInstance().getGraph();
+ long timeStart = System.nanoTime();
+
+ HashMap <String,String> old2NewVertIdMap = new <String,String> HashMap ();
+
+ // We're going to try loading in the vertices - without edges or properties
+ // using Separate threads
+
+ ExecutorService executor = Executors.newFixedThreadPool(fCount);
+ List<Future<HashMap<String,String>>> list = new ArrayList<Future<HashMap<String,String>>>();
+
+ for( int i=0; i < fCount; i++ ){
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ Thread.sleep(staggerThreadDelay); // Stagger the threads a bit
+ LOGGER.debug(" -- Read file: [" + fullSnapName + "]");
+ LOGGER.debug(" -- Call the PartialVertexLoader to just load vertices ----");
+ LOGGER.debug(" -- vertAddDelayMs = " + vertAddDelayMs
+ + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs
+ + ", maxErrorsPerThread = " + maxErrorsPerThread );
+ Callable <HashMap<String,String>> vLoader = new PartialVertexLoader(graph1, fullSnapName,
+ vertAddDelayMs, failureDelayMs, retryDelayMs, maxErrorsPerThread, LOGGER);
+ Future <HashMap<String,String>> future = (Future<HashMap<String, String>>) executor.submit(vLoader);
+
+ // add Future to the list, we can get return value using Future
+ list.add(future);
+ LOGGER.debug(" -- Starting PartialDbLoad VERT_ONLY thread # "+ i );
+ }
+
+ threadCount = 0;
+ int threadFailCount = 0;
+ for(Future<HashMap<String,String>> fut : list){
+ threadCount++;
+ try {
+ old2NewVertIdMap.putAll(fut.get());
+ LOGGER.debug(" -- back from PartialVertexLoader. returned thread # " + threadCount +
+ ", current size of old2NewVertMap is: " + old2NewVertIdMap.size() );
+ }
+ catch (InterruptedException e) {
+ threadFailCount++;
+ e.printStackTrace();
+ }
+ catch (ExecutionException e) {
+ threadFailCount++;
+ e.printStackTrace();
+ }
+ }
+
+ executor.shutdown();
+
+ if( threadFailCount > 0 ) {
+ String emsg = " FAILURE >> " + threadFailCount + " Vertex-loader thread(s) failed to complete successfully. ";
+ LOGGER.debug(emsg);
+ throw new Exception( emsg );
+ }
+
+ long timeX = System.nanoTime();
+ long diffTime = timeX - timeStart;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To reload just the vertex ids from the snapshot files, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ // Give the DB a little time to chew on all those vertices
+ Thread.sleep(vertToEdgeProcDelay);
+
+ // ----------------------------------------------------------------------------------------
+ LOGGER.debug("\n\n\n -- Now do the edges/props ----------------------");
+ // ----------------------------------------------------------------------------------------
+
+
+ // We're going to try loading in the edges and missing properties
+ // Note - we're passing the whole oldVid2newVid mapping to the PartialPropAndEdgeLoader
+ // so that the String-updates to the GraphSON will happen in the threads instead of
+ // here in the un-threaded calling method.
+ executor = Executors.newFixedThreadPool(fCount);
+ ArrayList<Future<ArrayList<String>>> listEdg = new ArrayList<Future<ArrayList<String>>>();
+ for( int i=0; i < fCount; i++ ){
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ String fullSnapName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ Thread.sleep(staggerThreadDelay); // Stagger the threads a bit
+ LOGGER.debug(" -- Read file: [" + fullSnapName + "]");
+ LOGGER.debug(" -- Call the PartialPropAndEdgeLoader for Properties and EDGEs ----");
+ LOGGER.debug(" -- edgeAddDelayMs = " + vertAddDelayMs
+ + ", failureDelayMs = " + failureDelayMs + ", retryDelayMs = " + retryDelayMs
+ + ", maxErrorsPerThread = " + maxErrorsPerThread );
+
+ Callable eLoader = new PartialPropAndEdgeLoader(graph1, fullSnapName,
+ edgeAddDelayMs, failureDelayMs, retryDelayMs,
+ old2NewVertIdMap, maxErrorsPerThread, LOGGER);
+ Future <ArrayList<String>> future = (Future<ArrayList<String>>) executor.submit(eLoader);
+
+ //add Future to the list, we can get return value using Future
+ listEdg.add(future);
+ LOGGER.debug(" -- Starting PartialPropAndEdge thread # "+ i );
+ }
+
+ threadCount = 0;
+ for(Future<ArrayList<String>> fut : listEdg){
+ threadCount++;
+ try{
+ fut.get(); // DEBUG -- should be doing something with the return value if it's not empty - ie. errors
+ LOGGER.debug(" -- back from PartialPropAndEdgeLoader. thread # " + threadCount );
+ }
+ catch (InterruptedException e) {
+ threadFailCount++;
+ e.printStackTrace();
+ }
+ catch (ExecutionException e) {
+ threadFailCount++;
+ e.printStackTrace();
+ }
+ }
+
+ executor.shutdown();
+
+ if( threadFailCount > 0 ) {
+ String emsg = " FAILURE >> " + threadFailCount + " Property/Edge-loader thread(s) failed to complete successfully. ";
+ LOGGER.debug(emsg);
+ throw new Exception( emsg );
+ }
+
+ // This is needed so we can see the data committed by the called threads
+ graph1.tx().commit();
+
+ long timeEnd = System.nanoTime();
+ diffTime = timeEnd - timeX;
+ minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To reload the edges and properties from snapshot files, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+ long totalDiffTime = timeEnd - timeStart;
+ long totalMinCount = TimeUnit.NANOSECONDS.toMinutes(totalDiffTime);
+ long totalSecCount = TimeUnit.NANOSECONDS.toSeconds(totalDiffTime) - (60 * totalMinCount);
+ LOGGER.debug(" -- TOTAL multi-threaded reload time: " +
+ totalMinCount + " minutes, " + totalSecCount + " seconds " );
+
+ } else if (command.equals("CLEAR_ENTIRE_DATABASE")) {
+ // ------------------------------------------------------------------
+ // They are calling this to clear the db before re-loading it
+ // later
+ // ------------------------------------------------------------------
+
+ // First - make sure the backup file(s) they will be using can be
+ // found and has(have) data.
+ // getFilesToProcess makes sure the file(s) exist and have some data.
+ getFilesToProcess(targetDir, oldSnapshotFileName, true);
+
+ LOGGER.debug("\n>>> WARNING <<<< ");
+ LOGGER.debug(">>> All data and schema in this database will be removed at this point. <<<");
+ LOGGER.debug(">>> Processing will begin in 5 seconds. <<<");
+ LOGGER.debug(">>> WARNING <<<< ");
+
+ try {
+ // Give them a chance to back out of this
+ Thread.sleep(5000);
+ } catch (java.lang.InterruptedException ie) {
+ LOGGER.debug(" DB Clearing has been aborted. ");
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ LOGGER.debug(" Begin clearing out old data. ");
+ String rtConfig = AAIConstants.REALTIME_DB_CONFIG;
+ String serviceName = System.getProperty("aai.service.name", "NA");
+ LOGGER.debug("Getting new configs for clearig");
+ PropertiesConfiguration propertiesConfiguration = new AAIGraphConfig.Builder(rtConfig).forService(serviceName).withGraphType(REALTIME_DB).buildConfiguration();
+ if(isExistingTitan){
+ LOGGER.debug("Existing DB is Titan");
+ propertiesConfiguration.setProperty("graph.titan-version","1.0.0");
+ }
+ LOGGER.debug("Open New Janus Graph");
+ JanusGraph janusGraph = JanusGraphFactory.open(propertiesConfiguration);
+ verifyGraph(janusGraph);
+
+ if(isExistingTitan){
+ JanusGraphFactory.drop(janusGraph);
+ } else {
+ janusGraph.close();
+ JanusGraphCleanup.clear(janusGraph);
+ }
+ LOGGER.debug(" Done clearing data. ");
+ LOGGER.debug(">>> IMPORTANT - NOTE >>> you need to run the SchemaGenerator (use GenTester) before ");
+ LOGGER.debug(" reloading data or the data will be put in without indexes. ");
+ dbClearFlag = true;
+ LOGGER.debug("All done clearing DB");
+
+ } else if (command.equals("RELOAD_LEGACY_DATA")) {
+ // -------------------------------------------------------------------
+ // They want to restore the database from an old snapshot file
+ // -------------------------------------------------------------------
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ graph = AAIGraph.getInstance().getGraph();
+ if (oldSnapshotFileName.equals("")) {
+ String emsg = "No oldSnapshotFileName passed to DataSnapshot when RELOAD_LEGACY_DATA used.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ String oldSnapshotFullFname = targetDir + AAIConstants.AAI_FILESEP + oldSnapshotFileName;
+ File f = new File(oldSnapshotFullFname);
+ if (!f.exists()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be found.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ } else if (!f.canRead()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be read.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ } else if (f.length() == 0) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " had no data.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ LOGGER.debug("We will load data IN from the file = " + oldSnapshotFullFname);
+ LOGGER.debug(" Begin reloading JanusGraph 0.5 data. ");
+
+ LegacyGraphSONReader lgr = LegacyGraphSONReader.build().create();
+ InputStream is = new FileInputStream(oldSnapshotFullFname);
+ lgr.readGraph(is, graph);
+
+ LOGGER.debug("Completed the inputGraph command, now try to commit()... ");
+ graph.tx().commit();
+ LOGGER.debug("Completed reloading JanusGraph 0.5 data.");
+
+ long vCount = graph.traversal().V().count().next();
+ LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db.");
+ } else if (command.equals("RELOAD_DATA")) {
+ // -------------------------------------------------------------------
+ // They want to restore the database from an old snapshot file
+ // -------------------------------------------------------------------
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ graph = AAIGraph.getInstance().getGraph();
+ if (oldSnapshotFileName.equals("")) {
+ String emsg = "No oldSnapshotFileName passed to DataSnapshot when RELOAD_DATA used.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ String oldSnapshotFullFname = targetDir + AAIConstants.AAI_FILESEP + oldSnapshotFileName;
+ File f = new File(oldSnapshotFullFname);
+ if (!f.exists()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be found.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ } else if (!f.canRead()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be read.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ } else if (f.length() == 0) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " had no data.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ LOGGER.debug("We will load data IN from the file = " + oldSnapshotFullFname);
+ LOGGER.debug(" Begin reloading data. ");
+ graph.io(IoCore.graphson()).readGraph(oldSnapshotFullFname);
+ LOGGER.debug("Completed the inputGraph command, now try to commit()... ");
+ graph.tx().commit();
+ LOGGER.debug("Completed reloading data.");
+
+ long vCount = graph.traversal().V().count().next();
+
+ LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db.");
+
+ } else if (command.equals("RELOAD_DATA_MULTI")) {
+ // -------------------------------------------------------------------
+ // They want to restore the database from a group of snapshot files
+ // Note - this uses multiple snapshot files, but runs single-threaded.
+ // -------------------------------------------------------------------
+ verifyGraph(AAIGraph.getInstance().getGraph());
+ graph = AAIGraph.getInstance().getGraph();
+
+ ArrayList <File> snapFilesArr = getFilesToProcess(targetDir, oldSnapshotFileName, false);
+
+ long timeA = System.nanoTime();
+
+ int fCount = snapFilesArr.size();
+ Iterator <File> fItr = snapFilesArr.iterator();
+ Vector<InputStream> inputStreamsV = new Vector<>();
+ for( int i = 0; i < fCount; i++ ){
+ File f = snapFilesArr.get(i);
+ String fname = f.getName();
+ if (!f.canRead()) {
+ String emsg = "oldSnapshotFile " + fname + " could not be read.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ } else if (f.length() == 0) {
+ String emsg = "oldSnapshotFile " + fname + " had no data.";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + fname;
+ InputStream fis = new FileInputStream(fullFName);
+ inputStreamsV.add(fis);
+ }
+ // Now add inputStreams.elements() to the Vector,
+ // inputStreams.elements() will return Enumerations
+ InputStream sis = new SequenceInputStream(inputStreamsV.elements());
+ LOGGER.debug("Begin loading data from " + fCount + " files -----");
+ graph.io(IoCore.graphson()).reader().create().readGraph(sis, graph);
+ LOGGER.debug("Completed the inputGraph command, now try to commit()... ");
+ graph.tx().commit();
+ LOGGER.debug(" >> Completed reloading data.");
+
+ long vCount = graph.traversal().V().count().next();
+ LOGGER.debug("A little after repopulating from an old snapshot, we see: " + vCount + " vertices in the db.");
+
+ long timeB = System.nanoTime();
+ long diffTime = timeB - timeA;
+ long minCount = TimeUnit.NANOSECONDS.toMinutes(diffTime);
+ long secCount = TimeUnit.NANOSECONDS.toSeconds(diffTime) - (60 * minCount);
+ LOGGER.debug(" -- To Reload this snapshot, it took: " +
+ minCount + " minutes, " + secCount + " seconds " );
+
+
+ } else {
+ String emsg = "Bad command passed to DataSnapshot: [" + command + "]";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ } catch (AAIException e) {
+ ErrorLogHelper.logError("AAI_6128", e.getMessage());
+ LOGGER.error("Encountered an exception during the datasnapshot: ", e);
+ e.printStackTrace();
+ success = false;
+ } catch (Exception ex) {
+ ErrorLogHelper.logError("AAI_6128", ex.getMessage());
+ LOGGER.error("Encountered an exception during the datasnapshot: ", ex);
+ ex.printStackTrace();
+ success = false;
+ } finally {
+ if (!dbClearFlag && graph != null) {
+ // Any changes that worked correctly should have already done
+ // thier commits.
+ if(!"true".equals(System.getProperty("org.onap.aai.graphadmin.started"))) {
+ if (graph.isOpen()) {
+ graph.tx().rollback();
+ graph.close();
+ }
+ }
+ }
+ try {
+ baos.close();
+ } catch (IOException iox) {
+ }
+ }
+
+ if(success){
+ AAISystemExitUtil.systemExitCloseAAIGraph(0);
+ } else {
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ }// End of main()
+
+
+ private static ArrayList <File> getFilesToProcess(String targetDir, String oldSnapshotFileName, boolean doingClearDb)
+ throws Exception {
+
+ if( oldSnapshotFileName == null || oldSnapshotFileName.equals("") ){
+ String emsg = "No oldSnapshotFileName passed to DataSnapshot for Reload. ";
+ if( doingClearDb ) {
+ emsg = "No oldSnapshotFileName passed to DataSnapshot. Needed when Clearing the db in case we need a backup. ";
+ }
+ LOGGER.debug(emsg);
+ throw new Exception( emsg );
+ }
+
+ ArrayList <File> snapFilesArrList = new ArrayList <File> ();
+
+ // First, we'll assume that this is a multi-file snapshot and
+ // look for names based on that.
+ String thisSnapPrefix = oldSnapshotFileName + ".P";
+ File fDir = new File(targetDir); // Snapshot directory
+ File[] allFilesArr = fDir.listFiles();
+ for (File snapFile : allFilesArr) {
+ String snapFName = snapFile.getName();
+ if( snapFName.startsWith(thisSnapPrefix)){
+ if (!snapFile.canRead()) {
+ String emsg = "oldSnapshotFile " + snapFName + " could not be read.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ } else if (snapFile.length() == 0) {
+ String emsg = "oldSnapshotFile " + snapFName + " had no data.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ }
+ snapFilesArrList.add(snapFile);
+ }
+ }
+
+ if( snapFilesArrList.isEmpty() ){
+ // Multi-file snapshot check did not find files, so this may
+ // be a single-file snapshot.
+ String oldSnapshotFullFname = targetDir + AAIConstants.AAI_FILESEP + oldSnapshotFileName;
+ File f = new File(oldSnapshotFullFname);
+ if (!f.exists()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be found.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ } else if (!f.canRead()) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " could not be read.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ } else if (f.length() == 0) {
+ String emsg = "oldSnapshotFile " + oldSnapshotFullFname + " had no data.";
+ LOGGER.debug(emsg);
+ throw new Exception (emsg);
+ }
+ snapFilesArrList.add(f);
+ }
+
+ if( snapFilesArrList.isEmpty() ){
+ // Still haven't found anything.. that was not a good file name.
+ String fullFName = targetDir + AAIConstants.AAI_FILESEP + thisSnapPrefix;
+ String emsg = "oldSnapshotFile " + fullFName + "* could not be found.";
+ LOGGER.debug(emsg);
+ throw new Exception(emsg);
+ }
+
+ return snapFilesArrList;
+ }
+
+
+ public static void verifyGraph(JanusGraph graph) {
+
+ if (graph == null) {
+ String emsg = "Not able to get a graph object in DataSnapshot.java\n";
+ LOGGER.debug(emsg);
+ AAISystemExitUtil.systemExitCloseAAIGraph(1);
+ }
+
+ }
+
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java b/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java
new file mode 100644
index 0000000..cc9ca97
--- /dev/null
+++ b/src/main/java/org/onap/aai/datasnapshot/DataSnapshotTasks.java
@@ -0,0 +1,115 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datasnapshot;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import org.onap.aai.datagrooming.DataGrooming;
+import org.onap.aai.datagrooming.DataGroomingTasks;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.util.AAIConfig;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+@Component
+@PropertySource("file:${server.local.startpath}/etc/appprops/datatoolscrons.properties")
+public class DataSnapshotTasks {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DataSnapshotTasks.class);
+ private static final SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
+
+ @Scheduled(cron = "${datasnapshottasks.cron}" )
+ public void snapshotScheduleTask() throws AAIException, Exception {
+
+ LoggingContext.init();
+ LoggingContext.requestId(UUID.randomUUID().toString());
+ LoggingContext.partnerName("AAI");
+ LoggingContext.targetEntity("CronApp");
+ LoggingContext.component("dataSnapshot");
+ LoggingContext.serviceName("snapshotScheduleTask");
+ LoggingContext.targetServiceName("snapshotScheduleTask");
+ LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
+
+ if(!"true".equals(AAIConfig.get("aai.disable.check.snapshot.running", "false"))){
+ if(checkIfDataSnapshotIsRunning()){
+ LOGGER.info("Data Snapshot is already running on the system");
+ return;
+ }
+ }
+
+ LOGGER.info("Started cron job dataSnapshot @ " + dateFormat.format(new Date()));
+ try {
+ if (AAIConfig.get("aai.cron.enable.dataSnapshot").equals("true")) {
+ DataSnapshot dataSnapshot = new DataSnapshot();
+ String [] dataSnapshotParms = AAIConfig.get("aai.datasnapshot.params", "JUST_TAKE_SNAPSHOT").split("\\s+");
+ LOGGER.info("DataSnapshot Params {}", Arrays.toString(dataSnapshotParms));
+ dataSnapshot.main(dataSnapshotParms);
+ }
+ }
+ catch (Exception e) {
+ ErrorLogHelper.logError("AAI_4000", "Exception running cron job for DataSnapshot"+e.toString());
+ LOGGER.info("AAI_4000", "Exception running cron job for DataSnapshot"+e.toString());
+ throw e;
+ } finally {
+ LOGGER.info("Ended cron job dataSnapshot @ " + dateFormat.format(new Date()));
+ LoggingContext.clear();
+ }
+
+ }
+
+ private boolean checkIfDataSnapshotIsRunning(){
+
+ Process process = null;
+
+ int count = 0;
+ try {
+ process = new ProcessBuilder().command("bash", "-c", "ps -ef | grep '[D]ataSnapshot'").start();
+ InputStream is = process.getInputStream();
+ InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader br = new BufferedReader(isr);
+
+ while (br.readLine() != null){
+ count++;
+ }
+
+ int exitVal = process.waitFor();
+ LOGGER.info("Exit value of the dataSnapshot check process: " + exitVal);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ if(count > 0){
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
+
+ \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java
new file mode 100644
index 0000000..af858ae
--- /dev/null
+++ b/src/main/java/org/onap/aai/datasnapshot/PartialPropAndEdgeLoader.java
@@ -0,0 +1,421 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datasnapshot;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.concurrent.Callable;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import com.att.eelf.configuration.EELFLogger;
+
+
+
+
+public class PartialPropAndEdgeLoader implements Callable <ArrayList<String>>{
+
+ private EELFLogger LOGGER;
+
+ private JanusGraph jg;
+ private String fName;
+ private Long edgeAddDelayMs;
+ private Long retryDelayMs;
+ private Long failureDelayMs;
+ private HashMap<String,String> old2NewVidMap;
+ private int maxAllowedErrors;
+
+
+
+ public PartialPropAndEdgeLoader (JanusGraph graph, String fn, Long edgeDelay, Long failureDelay, Long retryDelay,
+ HashMap<String,String> vidMap, int maxErrors, EELFLogger elfLog ){
+ jg = graph;
+ fName = fn;
+ edgeAddDelayMs = edgeDelay;
+ failureDelayMs = failureDelay;
+ retryDelayMs = retryDelay;
+ old2NewVidMap = vidMap;
+ maxAllowedErrors = maxErrors;
+ LOGGER = elfLog;
+ }
+
+
+ public ArrayList<String> call() throws Exception {
+
+ // This is a partner to the "PartialVertexLoader" code.
+ // That code loads in vertex-id's/vertex-label's for a
+ // multi-file data snapshot.
+ // This code assumes that the all vertex-id's are now in the target db.
+ // This code loads vertex properties and edges for a
+ // multi-file data snapshot (the same one that loaded
+ // the vertex-ids).
+ //
+
+
+ // NOTE - We will be loading parameters and edges for one node at a time so that problems can be
+ // identified or ignored or re-tried instead of causing the entire load to fail.
+ //
+ // Return an arrayList of Strings to give info on what nodes encountered problems
+
+ int entryCount = 0;
+ int retryCount = 0;
+ int failureCount = 0;
+ int retryFailureCount = 0;
+ HashMap <String,String> failedAttemptHash = new HashMap <String,String> ();
+ ArrayList <String> failedAttemptInfo = new ArrayList <String> ();
+
+ int passNum = 1;
+ try( BufferedReader br = new BufferedReader(new FileReader(fName))) {
+ // loop through the file lines and do PUT for each vertex or the edges depending on what the loadtype is
+ for(String origLine; (origLine = br.readLine()) != null; ) {
+ entryCount++;
+ Thread.sleep(edgeAddDelayMs); // Space the edge requests out a little
+
+ String errInfoStr = processThisLine(origLine, passNum);
+ if( !errInfoStr.equals("") ){
+ // There was a problem with this line
+ String vidStr = getTheVidForThisLine(origLine);
+ // We'll use the failedAttemptHash to reTry this item
+ failedAttemptHash.put(vidStr,origLine);
+ failedAttemptInfo.add(errInfoStr);
+ failureCount++;
+ if( failureCount > maxAllowedErrors ) {
+ LOGGER.debug(">>> Abandoning PartialPropAndEdgeLoader() because " +
+ "Max Allowed Error count was exceeded for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ Thread.sleep(failureDelayMs); // take a little nap if it failed
+ }
+ } // End of looping over each line
+ if( br != null ){
+ br.close();
+ }
+ }
+ catch (Exception e) {
+ LOGGER.debug(" --- Failed in the main loop for Buffered-Reader item # " + entryCount +
+ ", fName = " + fName );
+ LOGGER.debug(" --- msg = " + e.getMessage() );
+ throw e;
+ }
+
+ // ---------------------------------------------------------------------------
+ // Now Re-Try any failed requests that might have Failed on the first pass.
+ // ---------------------------------------------------------------------------
+ passNum++;
+ try {
+ for (String failedVidStr : failedAttemptHash.keySet()) {
+ // Take a little nap, and retry this failed attempt
+ LOGGER.debug("DEBUG >> We will sleep for " + retryDelayMs + " and then RETRY any failed edge/property ADDs. ");
+ Thread.sleep(retryDelayMs);
+ retryCount++;
+ Long failedVidL = Long.parseLong(failedVidStr);
+ // When an Edge/Property Add fails, we store the whole (translated) graphSON line as the data in the failedAttemptHash
+ // We're really just doing a GET of this one vertex here...
+ String jsonLineToRetry = failedAttemptHash.get(failedVidStr);
+ String errInfoStr = processThisLine(jsonLineToRetry, passNum);
+ if( !errInfoStr.equals("") ){
+ // There was a problem with this line
+ String translatedVidStr = getTheVidForThisLine(jsonLineToRetry);
+ failedAttemptHash.put(translatedVidStr,jsonLineToRetry);
+ failedAttemptInfo.add(errInfoStr);
+ retryFailureCount++;
+ if( retryFailureCount > maxAllowedErrors ) {
+ LOGGER.debug(">>> Abandoning PartialPropAndEdgeLoader() because " +
+ "Max Allowed Error count was exceeded while doing retries for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ Thread.sleep(failureDelayMs); // take a little nap if it failed
+ }
+ } // End of looping over each failed line
+ }
+ catch (Exception e) {
+ LOGGER.debug(" -- error in RETRY block. ErrorMsg = [" + e.getMessage() + "]" );
+ throw e;
+ }
+
+ LOGGER.debug(">>> After Processing in PartialPropAndEdgeLoader() " +
+ entryCount + " records processed. " + failureCount + " records failed. " +
+ retryCount + " RETRYs processed. " + retryFailureCount + " RETRYs failed. ");
+
+ return failedAttemptInfo;
+
+ }// end of call()
+
+
+
+ private String translateThisVid(String oldVid) throws Exception {
+
+ if( old2NewVidMap == null ){
+ throw new Exception(" ERROR - null old2NewVidMap found in translateThisVid. ");
+ }
+
+ if( old2NewVidMap.containsKey(oldVid) ){
+ return old2NewVidMap.get(oldVid);
+ }
+ else {
+ throw new Exception(" ERROR - could not find VID translation for original VID = " + oldVid );
+ }
+ }
+
+
+ private String getTheVidForThisLine(String graphSonLine) throws Exception {
+
+ if( graphSonLine == null ){
+ throw new Exception(" ERROR - null graphSonLine passed to getTheVidForThisLine. ");
+ }
+
+ // We are assuming that the graphSonLine has the vertexId as the first ID:
+ // {"id":100995128,"label":"vertex","inE":{"hasPinterface":[{"id":"7lgg0e-2... etc...
+
+ // The vertexId for this line is the numeric part after the initial {"id":xxxxx up to the first comma
+ int x = graphSonLine.indexOf(':') + 1;
+ int y = graphSonLine.indexOf(',');
+ String initialVid = graphSonLine.substring(x,y);
+ if( initialVid != null && !initialVid.isEmpty() && initialVid.matches("^[0-9]+$") ){
+ return initialVid;
+ }
+ else {
+ throw new Exception(" ERROR - could not determine initial VID for graphSonLine: " + graphSonLine );
+ }
+ }
+
+
+ private String processThisLine(String graphSonLine, int passNum){
+
+ String passInfo = "";
+ if( passNum > 1 ) {
+ passInfo = " >> RETRY << pass # " + passNum + " ";
+ }
+
+ JSONObject jObj = new JSONObject();
+ String originalVid = "";
+
+ try{
+ jObj = new JSONObject(graphSonLine);
+ originalVid = jObj.get("id").toString();
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- Could not convert line to JsonObject [ " + graphSonLine + "]" );
+ LOGGER.debug(" -- ErrorMsg = [" +e.getMessage() + "]");
+
+ return(" DEBUG -a- JSON translation exception when processing this line ---");
+ //xxxxxDEBUGxxxxx I think we put some info on the return String and then return?
+ }
+
+ // -----------------------------------------------------------------------------------------
+ // Note - this assumes that any vertices referred to by an edge will already be in the DB.
+ // -----------------------------------------------------------------------------------------
+ Vertex dbVtx = null;
+
+ String newVidStr = "";
+ Long newVidL = 0L;
+ try {
+ newVidStr = translateThisVid(originalVid);
+ newVidL = Long.parseLong(newVidStr);
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " translate VertexId before adding edges failed for this: vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]");
+
+ return(" DEBUG -b- there VID-translation error when processing this line ---");
+ //xxxxxDEBUGxxxxx I think we put some info on the return String and then return?
+ }
+
+
+ try {
+ dbVtx = getVertexFromDbForVid(newVidStr);
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " READ Vertex from DB before adding edges failed for this: vtxId = " + originalVid
+ + ", newVidId = " + newVidL + ". ErrorMsg = [" +e.getMessage() + "]");
+
+ return(" -- there was an error processing this line --- Line = [" + graphSonLine + "]");
+ //xxxxxxDEBUGxxxx I think we put some info on the return String and then return?
+ }
+
+
+ String edResStr = processEdgesForVtx( jObj, dbVtx, passInfo, originalVid );
+ if( edResStr.equals("") ){
+ // We will commit the edges by themselves in case the properties stuff below fails
+ try {
+ jg.tx().commit();
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " COMMIT FAILED adding EDGES for this vertex: vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]");
+ //xxxxxxxxxx I think we put some info on the return String and then return?
+ return(" DEBUG -d- there was an error doing the commit while processing edges for this line ---");
+ }
+ }
+
+ // Add the properties that we didn't have when we added the 'bare-bones' vertex
+ String pResStr = processPropertiesForVtx( jObj, dbVtx, passInfo, originalVid );
+ if( pResStr.equals("") ){
+ try {
+ jg.tx().commit();
+ return "";
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " COMMIT FAILED adding Properties for this vertex: vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]");
+ //xxxxxxxxxx I think we put some info on the return String and then return?
+ return(" DEBUG -e- there was an error doing the commit while processing Properties for this line ---");
+ }
+ }
+ else {
+ LOGGER.debug("DEBUG " + passInfo + " Error processing Properties for this vertex: vtxId = " + originalVid );
+
+ //xxxxxxxxxx I think we put some info on the return String and then return?
+ return(" DEBUG -f- there was an error while processing Properties for this line ---");
+ }
+ }
+
+
+ private String processPropertiesForVtx( JSONObject jObj, Vertex dbVtx, String passInfo, String originalVid ){
+
+ try {
+ JSONObject propsOb = (JSONObject) jObj.get("properties");
+ Iterator <String> propsItr = propsOb.keys();
+ while( propsItr.hasNext() ){
+ String pKey = propsItr.next();
+ JSONArray propsDetArr = propsOb.getJSONArray(pKey);
+ for( int i=0; i< propsDetArr.length(); i++ ){
+ JSONObject prop = propsDetArr.getJSONObject(i);
+ String val = prop.getString("value");
+ dbVtx.property(pKey, val); //DEBUGjojo -- val is always String here.. which is not right -------------------DEBUG
+ }
+ }
+
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- " + passInfo + " failure getting/setting properties for: vtxId = "
+ + originalVid + ". ErrorMsg = [" + e.getMessage() + "]");
+ //xxxDEBUGxxxxxxx I think we put some info on the return String and then return?
+ return(" DEBUG -g- there was an error adding properties while processing this line ---");
+
+ }
+
+ return "";
+ }
+
+
+ private Vertex getVertexFromDbForVid( String vtxIdStr ) throws Exception {
+ Vertex thisVertex = null;
+ Long vtxIdL = 0L;
+
+ try {
+ vtxIdL = Long.parseLong(vtxIdStr);
+ Iterator <Vertex> vItr = jg.vertices(vtxIdL);
+ // Note - we only expect to find one vertex found for this ID.
+ while( vItr.hasNext() ){
+ thisVertex = vItr.next();
+ }
+ }
+ catch ( Exception e ){
+ String emsg = "Error finding vertex for vid = " + vtxIdStr + "[" + e.getMessage() + "]";
+ throw new Exception ( emsg );
+ }
+
+ if( thisVertex == null ){
+ String emsg = "Could not find vertex for passed vid = " + vtxIdStr;
+ throw new Exception ( emsg );
+ }
+
+ return thisVertex;
+ }
+
+
+ private String processEdgesForVtx( JSONObject jObj, Vertex dbVtx, String passInfo, String originalVid ){
+
+ // Process the edges for this vertex -- but, just the "OUT" ones so edges don't get added twice (once from
+ // each side of the edge).
+ JSONObject edOb = null;
+ try {
+ edOb = (JSONObject) jObj.get("outE");
+ }
+ catch (Exception e){
+ // There were no OUT edges. This is OK.
+ return "";
+ }
+
+ try {
+ if( edOb == null ){
+ // There were no OUT edges. This is OK. Not all nodes have out edges.
+ return "";
+ }
+ Iterator <String> edItr = edOb.keys();
+ while( edItr.hasNext() ){
+ String eLabel = edItr.next();
+ String inVid = ""; // Note - this should really be a Long?
+ JSONArray edArr = edOb.getJSONArray(eLabel);
+ for( int i=0; i< edArr.length(); i++ ){
+ JSONObject eObj = edArr.getJSONObject(i);
+ String inVidStr = eObj.get("inV").toString();
+ String translatedInVidStr = translateThisVid(inVidStr);
+ Vertex newInVertex = getVertexFromDbForVid(translatedInVidStr);
+
+ // Note - addEdge automatically adds the edge in the OUT direction from the
+ // 'anchor' node that the call is being made from.
+ Edge tmpE = dbVtx.addEdge(eLabel, newInVertex);
+ JSONObject ePropsOb = null;
+ try {
+ ePropsOb = (JSONObject) eObj.get("properties");
+ }
+ catch (Exception e){
+ // NOTE - model definition related edges do not have edge properties. That is OK.
+ // Ie. when a model-element node has an "isA" edge to a "model-ver" node, that edge does
+ // not have edge properties on it.
+ }
+ if( ePropsOb != null ){
+ Iterator <String> ePropsItr = ePropsOb.keys();
+ while( ePropsItr.hasNext() ){
+ String pKey = ePropsItr.next();
+ tmpE.property(pKey, ePropsOb.getString(pKey));
+ }
+ }
+ }
+ }
+
+ }
+ catch ( Exception e ){
+ String msg = " -- " + passInfo + " failure adding edge for: original vtxId = "
+ + originalVid + ". ErrorMsg = [" +e.getMessage() + "]";
+ LOGGER.debug( " -- " + msg );
+ //xxxxxxDEBUGxxxx I think we might need some better info on the return String to return?
+ LOGGER.debug(" -- now going to return/bail out of processEdgesForVtx" );
+ return(" >> " + msg );
+
+ }
+
+ return "";
+ }
+
+
+}
+
+
diff --git a/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java b/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java
new file mode 100644
index 0000000..387f45e
--- /dev/null
+++ b/src/main/java/org/onap/aai/datasnapshot/PartialVertexLoader.java
@@ -0,0 +1,223 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datasnapshot;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.HashMap;
+import java.util.concurrent.Callable;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader;
+import org.janusgraph.core.JanusGraph;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+
+
+public class PartialVertexLoader implements Callable<HashMap<String,String>>{
+
+ private EELFLogger LOGGER;
+
+ private JanusGraph jg;
+ private String fName;
+ private Long vertAddDelayMs;
+ private Long failurePauseMs;
+ private Long retryDelayMs;
+ private int maxAllowedErrors;
+
+ public PartialVertexLoader (JanusGraph graph, String fn, Long vertDelay, Long failurePause,
+ Long retryDelay, int maxErrors, EELFLogger elfLog ){
+ jg = graph;
+ fName = fn;
+ vertAddDelayMs = vertDelay;
+ failurePauseMs = failurePause;
+ retryDelayMs = retryDelay;
+ maxAllowedErrors = maxErrors;
+ LOGGER = elfLog;
+ }
+
+ public HashMap<String,String> call() throws Exception {
+
+ // NOTE - we will be loading one node at a time so that bad nodes can be ignored instead of causing the
+ // entire load to fail.
+ //
+ int entryCount = 0;
+ int retryCount = 0;
+ int failureCount = 0;
+ int retryFailureCount = 0;
+ HashMap <String,String> failedAttemptHash = new HashMap <String,String> ();
+ HashMap <String,String> old2NewVtxIdHash = new HashMap <String,String> ();
+ GraphSONReader gsr = GraphSONReader.build().create();
+
+
+ // Read this file into a JSON object
+ JsonParser parser = new JsonParser();
+
+ try( BufferedReader br = new BufferedReader(new FileReader(fName))) {
+ // loop through the file lines and do PUT for each vertex or the edges depending on what the loadtype is
+ for(String line; (line = br.readLine()) != null; ) {
+ entryCount++;
+ Object ob = parser.parse(line);
+ JsonObject jObj = (JsonObject) ob;
+ // NOTE - we will need to keep track of how the newly generated vid's map
+ // to the old ones so we can aim the edges correctly later.
+
+ // ---- Note -- This ONLY loads the vertexId and the label for each vertex -------------
+ Thread.sleep(vertAddDelayMs);
+
+ String oldVtxIdStr = jObj.get("id").getAsString();
+ String vtxLabelStr = jObj.get("label").getAsString();
+ try {
+ Vertex tmpV = jg.addVertex(vtxLabelStr);
+ String newVtxIdStr = tmpV.id().toString();
+ old2NewVtxIdHash.put(oldVtxIdStr, newVtxIdStr);
+ }
+ catch ( Exception e ){
+ failureCount++;
+ Thread.sleep(failurePauseMs); // Slow down if things are failing
+ LOGGER.debug(" >> addVertex FAILED for vtxId = " + oldVtxIdStr + ", label = ["
+ + vtxLabelStr + "]. ErrorMsg = [" + e.getMessage() + "]" );
+ //e.printStackTrace();
+ failedAttemptHash.put(oldVtxIdStr, vtxLabelStr);
+ if( failureCount > maxAllowedErrors ) {
+ LOGGER.debug(" >>> Abandoning PartialVertexLoader() because " +
+ "Max Allowed Error count was exceeded for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ else {
+ continue;
+ }
+ }
+ try {
+ jg.tx().commit();
+ }
+ catch ( Exception e ){
+ failureCount++;
+ Thread.sleep(failurePauseMs); // Slow down if things are failing
+ LOGGER.debug(" -- COMMIT FAILED for Vtx ADD for vtxId = " + oldVtxIdStr + ", label = ["
+ + vtxLabelStr + "]. ErrorMsg = [" +e.getMessage() + "]" );
+ //e.printStackTrace();
+ failedAttemptHash.put(oldVtxIdStr, vtxLabelStr);
+ if( failureCount > maxAllowedErrors ) {
+ LOGGER.debug(">>> Abandoning PartialVertexLoader() because " +
+ "Max Allowed Error count was exceeded for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ else {
+ continue;
+ }
+ }
+
+ } // End of looping over each line
+
+ if( br != null ){
+ br.close();
+ }
+ }
+ catch (Exception e) {
+ LOGGER.debug(" --- Failed in the main loop for Buffered-Reader item # " + entryCount +
+ ", fName = " + fName );
+ LOGGER.debug(" --- msg = " + e.getMessage() );
+ e.printStackTrace();
+ throw e;
+ }
+
+ // ---------------------------------------------------------------------------
+ // Now Re-Try any failed requests that might have Failed on the first pass.
+ // ---------------------------------------------------------------------------
+ try {
+ for (String failedVidStr : failedAttemptHash.keySet()) {
+ // Take a little nap, and retry this failed attempt
+ LOGGER.debug("DEBUG >> We will sleep for " + retryDelayMs + " and then RETRY any failed vertex ADDs. ");
+ Thread.sleep(retryDelayMs);
+
+ retryCount++;
+ // When a vertex Add fails we store the label as the data in the failedAttemptHash.
+ String failedLabel = failedAttemptHash.get(failedVidStr);
+ LOGGER.debug("DEBUG >> RETRY << " +
+ failedVidStr + ", label = " + failedLabel );
+ try {
+ Vertex tmpV = jg.addVertex(failedLabel);
+ String newVtxIdStr = tmpV.id().toString();
+ old2NewVtxIdHash.put(failedVidStr, newVtxIdStr);
+ }
+ catch ( Exception e ){
+ retryFailureCount++;
+ LOGGER.debug(" -- addVertex FAILED for RETRY for vtxId = " +
+ failedVidStr + ", label = [" + failedLabel +
+ "]. ErrorMsg = [" +e.getMessage() + "]" );
+ e.printStackTrace();
+ if( retryFailureCount > maxAllowedErrors ) {
+ LOGGER.debug(">>> Abandoning PartialVertexLoader() because " +
+ "Max Allowed Error count was exceeded for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ else {
+ continue;
+ }
+ }
+ try {
+ jg.tx().commit();
+ // If this worked, we can take it off of the failed list
+ failedAttemptHash.remove(failedVidStr);
+ }
+ catch ( Exception e ){
+ retryFailureCount++;
+ LOGGER.debug(" -- COMMIT FAILED for RETRY for vtxId = " + failedVidStr
+ + ", label = [" + failedLabel + "]. ErrorMsg = [" + e.getMessage() + "]" );
+ e.printStackTrace();
+ if( retryFailureCount > maxAllowedErrors ) {
+ LOGGER.debug(">>> Abandoning PartialVertexLoader() because " +
+ "Max Allowed Error count was exceeded for this thread. (max = " +
+ maxAllowedErrors + ". ");
+ throw new Exception(" ERROR - Max Allowed Error count exceeded for this thread. (max = " + maxAllowedErrors + ". ");
+ }
+ else {
+ continue;
+ }
+ }
+ } // End of looping over failed attempt hash and doing retries
+
+ }
+ catch ( Exception e ){
+ LOGGER.debug(" -- error in RETRY block. ErrorMsg = [" +e.getMessage() + "]" );
+ e.printStackTrace();
+ throw e;
+ }
+
+ // This would need to be properly logged...
+ LOGGER.debug(">>> After Processing in PartialVertexLoader(): " +
+ entryCount + " records processed. " + failureCount + " records failed. " +
+ retryCount + " RETRYs processed. " + retryFailureCount + " RETRYs failed. ");
+
+ return old2NewVtxIdHash;
+
+ }// end of call()
+
+
+
+}
+
+
diff --git a/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java b/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java
new file mode 100644
index 0000000..791ae15
--- /dev/null
+++ b/src/main/java/org/onap/aai/datasnapshot/PrintVertexDetails.java
@@ -0,0 +1,107 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.datasnapshot;
+
+import java.io.FileOutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.IoCore;
+import org.janusgraph.core.JanusGraph;
+
+
+public class PrintVertexDetails implements Runnable{
+
+ //private static EELFLogger LOGGER;
+
+ private JanusGraph jg;
+ private String fname;
+ private ArrayList<Vertex> vtxList;
+ private Boolean debugOn;
+ private int debugDelayMs;
+
+ public PrintVertexDetails (JanusGraph graph, String fn, ArrayList<Vertex> vL, Boolean debugFlag, int debugDelay){
+ jg = graph;
+ fname = fn;
+ vtxList = vL;
+ debugOn = debugFlag;
+ debugDelayMs = debugDelay;
+ }
+
+ public void run(){
+ if( debugOn ){
+ // This is much slower, but sometimes we need to find out which single line is causing a failure
+ try{
+ int okCount = 0;
+ int failCount = 0;
+ Long debugDelayMsL = new Long(debugDelayMs);
+ FileOutputStream subFileStr = new FileOutputStream(fname);
+ Iterator <Vertex> vSubItr = vtxList.iterator();
+ while( vSubItr.hasNext() ){
+ Long vertexIdL = 0L;
+ String aaiNodeType = "";
+ String aaiUri = "";
+ String aaiUuid = "";
+ try {
+ Vertex tmpV = vSubItr.next();
+ vertexIdL = (Long) tmpV.id();
+ aaiNodeType = (String) tmpV.property("aai-node-type").orElse(null);
+ aaiUri = (String) tmpV.property("aai-uri").orElse(null);
+ aaiUuid = (String) tmpV.property("aai-uuid").orElse(null);
+
+ Thread.sleep(debugDelayMsL); // Make sure it doesn't bump into itself
+ jg.io(IoCore.graphson()).writer().create().writeVertex(subFileStr, tmpV, Direction.BOTH);
+ okCount++;
+ }
+ catch(Exception e) {
+ failCount++;
+ System.out.println(" >> DEBUG MODE >> Failed at: VertexId = [" + vertexIdL +
+ "], aai-node-type = [" + aaiNodeType +
+ "], aai-uuid = [" + aaiUuid +
+ "], aai-uri = [" + aaiUri + "]. " );
+ e.printStackTrace();
+ }
+ }
+ System.out.println(" -- Printed " + okCount + " vertexes out to " + fname +
+ ", with " + failCount + " failed.");
+ subFileStr.close();
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ }
+ }
+ else {
+ // Not in DEBUG mode, so we'll do all the nodes in one group
+ try{
+ int count = vtxList.size();
+ Iterator <Vertex> vSubItr = vtxList.iterator();
+ FileOutputStream subFileStr = new FileOutputStream(fname);
+ jg.io(IoCore.graphson()).writer().create().writeVertices(subFileStr, vSubItr, Direction.BOTH);
+ subFileStr.close();
+ System.out.println(" -- Printed " + count + " vertexes out to " + fname);
+ }
+ catch(Exception e){
+ e.printStackTrace();
+ }
+ }
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/db/schema/AuditDoc.java b/src/main/java/org/onap/aai/db/schema/AuditDoc.java
new file mode 100644
index 0000000..2beec12
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/AuditDoc.java
@@ -0,0 +1,88 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+import java.util.List;
+
+public class AuditDoc {
+
+ private List<DBProperty> properties;
+ private List<DBIndex> indexes;
+ private List<EdgeProperty> edgeLabels;
+
+ /**
+ * Gets the properties.
+ *
+ * @return the properties
+ */
+ public List<DBProperty> getProperties() {
+ return properties;
+ }
+
+ /**
+ * Sets the properties.
+ *
+ * @param properties the new properties
+ */
+ public void setProperties(List<DBProperty> properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * Gets the indexes.
+ *
+ * @return the indexes
+ */
+ public List<DBIndex> getIndexes() {
+ return indexes;
+ }
+
+ /**
+ * Sets the indexes.
+ *
+ * @param indexes the new indexes
+ */
+ public void setIndexes(List<DBIndex> indexes) {
+ this.indexes = indexes;
+ }
+
+ /**
+ * Gets the edge labels.
+ *
+ * @return the edge labels
+ */
+ @JsonProperty("edge-labels")
+ public List<EdgeProperty> getEdgeLabels() {
+ return edgeLabels;
+ }
+
+ /**
+ * Sets the edge labels.
+ *
+ * @param edgeLabels the new edge labels
+ */
+ public void setEdgeLabels(List<EdgeProperty> edgeLabels) {
+ this.edgeLabels = edgeLabels;
+ }
+
+
+}
diff --git a/src/main/java/org/onap/aai/db/schema/AuditJanusGraph.java b/src/main/java/org/onap/aai/db/schema/AuditJanusGraph.java
new file mode 100644
index 0000000..e49aa7f
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/AuditJanusGraph.java
@@ -0,0 +1,121 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.EdgeLabel;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphIndex;
+import org.janusgraph.core.schema.JanusGraphManagement;
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+
+public class AuditJanusGraph extends Auditor {
+
+ private final JanusGraph graph;
+
+ /**
+ * Instantiates a new audit JanusGraph.
+ *
+ * @param g the g
+ */
+ public AuditJanusGraph (JanusGraph g) {
+ this.graph = g;
+ buildSchema();
+ }
+
+ /**
+ * Builds the schema.
+ */
+ private void buildSchema() {
+ populateProperties();
+ populateIndexes();
+ populateEdgeLabels();
+ }
+
+ /**
+ * Populate properties.
+ */
+ private void populateProperties() {
+ JanusGraphManagement mgmt = graph.openManagement();
+ Iterable<PropertyKey> iterable = mgmt.getRelationTypes(PropertyKey.class);
+ Iterator<PropertyKey> JanusGraphProperties = iterable.iterator();
+ PropertyKey propKey;
+ while (JanusGraphProperties.hasNext()) {
+ propKey = JanusGraphProperties.next();
+ DBProperty prop = new DBProperty();
+
+ prop.setName(propKey.name());
+ prop.setCardinality(propKey.cardinality());
+ prop.setTypeClass(propKey.dataType());
+
+ this.properties.put(prop.getName(), prop);
+ }
+ }
+
+ /**
+ * Populate indexes.
+ */
+ private void populateIndexes() {
+ JanusGraphManagement mgmt = graph.openManagement();
+ Iterable<JanusGraphIndex> iterable = mgmt.getGraphIndexes(Vertex.class);
+ Iterator<JanusGraphIndex> JanusGraphIndexes = iterable.iterator();
+ JanusGraphIndex JanusGraphIndex;
+ while (JanusGraphIndexes.hasNext()) {
+ JanusGraphIndex = JanusGraphIndexes.next();
+ if (JanusGraphIndex.isCompositeIndex()) {
+ DBIndex index = new DBIndex();
+ LinkedHashSet<DBProperty> dbProperties = new LinkedHashSet<>();
+ index.setName(JanusGraphIndex.name());
+ index.setUnique(JanusGraphIndex.isUnique());
+ PropertyKey[] keys = JanusGraphIndex.getFieldKeys();
+ for (PropertyKey key : keys) {
+ dbProperties.add(this.properties.get(key.name()));
+ }
+ index.setProperties(dbProperties);
+ index.setStatus(JanusGraphIndex.getIndexStatus(keys[0]));
+ this.indexes.put(index.getName(), index);
+ }
+ }
+ }
+
+ /**
+ * Populate edge labels.
+ */
+ private void populateEdgeLabels() {
+ JanusGraphManagement mgmt = graph.openManagement();
+ Iterable<EdgeLabel> iterable = mgmt.getRelationTypes(EdgeLabel.class);
+ Iterator<EdgeLabel> JanusGraphEdgeLabels = iterable.iterator();
+ EdgeLabel edgeLabel;
+ while (JanusGraphEdgeLabels.hasNext()) {
+ edgeLabel = JanusGraphEdgeLabels.next();
+ EdgeProperty edgeProperty = new EdgeProperty();
+
+ edgeProperty.setName(edgeLabel.name());
+ edgeProperty.setMultiplicity(edgeLabel.multiplicity());
+
+ this.edgeLabels.put(edgeProperty.getName(), edgeProperty);
+ }
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/db/schema/AuditOXM.java b/src/main/java/org/onap/aai/db/schema/AuditOXM.java
new file mode 100644
index 0000000..417824c
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/AuditOXM.java
@@ -0,0 +1,227 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.google.common.collect.Multimap;
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.Multiplicity;
+import org.janusgraph.core.schema.SchemaStatus;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.EdgeRule;
+import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.schema.enums.ObjectMetadata;
+import org.onap.aai.setup.SchemaVersion;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class AuditOXM extends Auditor {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(AuditOXM.class);
+
+ private Set<Introspector> allObjects;
+ private EdgeIngestor ingestor;
+
+ /**
+ * Instantiates a new audit OXM.
+ *
+ * @param version the version
+ */
+ public AuditOXM(LoaderFactory loaderFactory, SchemaVersion version) {
+
+ Loader loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, version);
+ Set<String> objectNames = getAllObjects(loader);
+ allObjects = new HashSet<>();
+ for (String key : objectNames) {
+ try {
+ final Introspector temp = loader.introspectorFromName(key);
+ allObjects.add(temp);
+ this.createDBProperties(temp);
+ } catch (AAIUnknownObjectException e) {
+ LOGGER.warn("Skipping audit for object " + key + " (Unknown Object) " + LogFormatTools.getStackTop(e));
+ }
+ }
+ for (Introspector temp : allObjects) {
+ this.createDBIndexes(temp);
+ }
+ try {
+ createEdgeLabels();
+ } catch (EdgeRuleNotFoundException e) {
+ LOGGER.warn("Skipping audit for version " + version + " due to " + LogFormatTools.getStackTop(e));
+ }
+
+ }
+
+ /**
+ * Gets the all objects.
+ *
+ * @param version the version
+ * @return the all objects
+ */
+ private Set<String> getAllObjects(Loader loader) {
+
+ Set<String> result = loader.getAllObjects().entrySet()
+ .stream()
+ .map(Map.Entry::getKey)
+ .collect(Collectors.toSet());
+
+ result.remove("EdgePropNames");
+ return result;
+ }
+
+ /**
+ * Creates the DB properties.
+ *
+ * @param temp the temp
+ */
+ private void createDBProperties(Introspector temp) {
+ Set<String> objectProperties = temp.getProperties();
+
+ for (String prop : objectProperties) {
+ if (!properties.containsKey(prop)) {
+ DBProperty dbProperty = new DBProperty();
+ dbProperty.setName(prop);
+ if (temp.isListType(prop)) {
+ dbProperty.setCardinality(Cardinality.SET);
+ if (temp.isSimpleGenericType(prop)) {
+ Class<?> clazz = null;
+ try {
+ clazz = Class.forName(temp.getGenericType(prop));
+ } catch (ClassNotFoundException e) {
+ clazz = Object.class;
+ }
+ dbProperty.setTypeClass(clazz);
+ properties.put(prop, dbProperty);
+ }
+ } else {
+ dbProperty.setCardinality(Cardinality.SINGLE);
+ if (temp.isSimpleType(prop)) {
+ Class<?> clazz = null;
+ try {
+ clazz = Class.forName(temp.getType(prop));
+ } catch (ClassNotFoundException e) {
+ clazz = Object.class;
+ }
+ dbProperty.setTypeClass(clazz);
+ properties.put(prop, dbProperty);
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Creates the DB indexes.
+ *
+ * @param temp the temp
+ */
+ private void createDBIndexes(Introspector temp) {
+ String uniqueProps = temp.getMetadata(ObjectMetadata.UNIQUE_PROPS);
+ String namespace = temp.getMetadata(ObjectMetadata.NAMESPACE);
+ if (uniqueProps == null) {
+ uniqueProps = "";
+ }
+ if (namespace == null) {
+ namespace = "";
+ }
+ boolean isTopLevel = namespace != "";
+ List<String> unique = Arrays.asList(uniqueProps.split(","));
+ Set<String> indexed = temp.getIndexedProperties();
+ Set<String> keys = temp.getKeys();
+
+ for (String prop : indexed) {
+ DBIndex dbIndex = new DBIndex();
+ LinkedHashSet<DBProperty> properties = new LinkedHashSet<>();
+ if (!this.indexes.containsKey(prop)) {
+ dbIndex.setName(prop);
+ dbIndex.setUnique(unique.contains(prop));
+ properties.add(this.properties.get(prop));
+ dbIndex.setProperties(properties);
+ dbIndex.setStatus(SchemaStatus.ENABLED);
+ this.indexes.put(prop, dbIndex);
+ }
+ }
+ if (keys.size() > 1 || isTopLevel) {
+ DBIndex dbIndex = new DBIndex();
+ LinkedHashSet<DBProperty> properties = new LinkedHashSet<>();
+ dbIndex.setName("key-for-" + temp.getDbName());
+ if (!this.indexes.containsKey(dbIndex.getName())) {
+ boolean isUnique = false;
+ if (isTopLevel) {
+ properties.add(this.properties.get(AAIProperties.NODE_TYPE));
+ }
+ for (String key : keys) {
+ properties.add(this.properties.get(key));
+
+ if (unique.contains(key) && !isUnique) {
+ isUnique = true;
+ }
+ }
+ dbIndex.setUnique(isUnique);
+ dbIndex.setProperties(properties);
+ dbIndex.setStatus(SchemaStatus.ENABLED);
+ this.indexes.put(dbIndex.getName(), dbIndex);
+ }
+ }
+
+ }
+
+ /**
+ * Creates the edge labels.
+ */
+ private void createEdgeLabels() throws EdgeRuleNotFoundException {
+ Multimap<String, EdgeRule> edgeRules = ingestor.getAllCurrentRules();
+ for (String key : edgeRules.keySet()) {
+ Collection<EdgeRule> collection = edgeRules.get(key);
+ EdgeProperty prop = new EdgeProperty();
+ //there is only ever one, they used the wrong type for EdgeRules
+ String label = "";
+ for (EdgeRule item : collection) {
+ label = item.getLabel();
+ }
+ prop.setName(label);
+ prop.setMultiplicity(Multiplicity.MULTI);
+ this.edgeLabels.put(label, prop);
+ }
+ }
+
+ /**
+ * Gets the all introspectors.
+ *
+ * @return the all introspectors
+ */
+ public Set<Introspector> getAllIntrospectors() {
+ return this.allObjects;
+ }
+
+ public void setEdgeIngestor(EdgeIngestor ingestor){
+ this.ingestor = ingestor;
+ }
+}
diff --git a/src/main/java/org/onap/aai/db/schema/Auditor.java b/src/main/java/org/onap/aai/db/schema/Auditor.java
new file mode 100644
index 0000000..5dc8c6c
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/Auditor.java
@@ -0,0 +1,53 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import java.util.*;
+
+public abstract class Auditor {
+
+ protected Map<String, DBProperty> properties = new HashMap<>();
+ protected Map<String, DBIndex> indexes = new HashMap<>();
+ protected Map<String, EdgeProperty> edgeLabels = new HashMap<>();
+
+ /**
+ * Gets the audit doc.
+ *
+ * @return the audit doc
+ */
+ public AuditDoc getAuditDoc() {
+ AuditDoc doc = new AuditDoc();
+ List<DBProperty> propertyList = new ArrayList<>();
+ List<DBIndex> indexList = new ArrayList<>();
+ List<EdgeProperty> edgeLabelList = new ArrayList<>();
+ propertyList.addAll(this.properties.values());
+ indexList.addAll(this.indexes.values());
+ edgeLabelList.addAll(this.edgeLabels.values());
+ Collections.sort(propertyList, new CompareByName());
+ Collections.sort(indexList, new CompareByName());
+ Collections.sort(edgeLabelList, new CompareByName());
+
+ doc.setProperties(propertyList);
+ doc.setIndexes(indexList);
+ doc.setEdgeLabels(edgeLabelList);
+
+ return doc;
+ }
+}
diff --git a/src/main/java/org/onap/aai/db/schema/AuditorFactory.java b/src/main/java/org/onap/aai/db/schema/AuditorFactory.java
new file mode 100644
index 0000000..6d96f29
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/AuditorFactory.java
@@ -0,0 +1,53 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import org.janusgraph.core.JanusGraph;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.setup.SchemaVersion;
+
+public class AuditorFactory {
+
+ private LoaderFactory loaderFactory;
+
+ public AuditorFactory(LoaderFactory loaderFactory){
+ this.loaderFactory = loaderFactory;
+ }
+ /**
+ * Gets the OXM auditor.
+ *
+ * @param v the v
+ * @return the OXM auditor
+ */
+ public Auditor getOXMAuditor (SchemaVersion v) {
+ return new AuditOXM(loaderFactory, v);
+ }
+
+ /**
+ * Gets the graph auditor.
+ *
+ * @param g the g
+ * @return the graph auditor
+ */
+ public Auditor getGraphAuditor (JanusGraph g) {
+ return new AuditJanusGraph(g);
+ }
+}
diff --git a/src/main/java/org/onap/aai/db/schema/CompareByName.java b/src/main/java/org/onap/aai/db/schema/CompareByName.java
new file mode 100644
index 0000000..829239d
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/CompareByName.java
@@ -0,0 +1,35 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import java.util.Comparator;
+
+public class CompareByName implements Comparator<Named>{
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public int compare(Named o1, Named o2) {
+ return o1.getName().compareTo(o2.getName());
+ }
+
+
+}
diff --git a/src/main/java/org/onap/aai/db/schema/DBIndex.java b/src/main/java/org/onap/aai/db/schema/DBIndex.java
new file mode 100644
index 0000000..754999c
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/DBIndex.java
@@ -0,0 +1,104 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import org.janusgraph.core.schema.SchemaStatus;
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+public class DBIndex implements Named {
+
+ private String name = null;
+ private boolean unique = false;
+ private LinkedHashSet<DBProperty> properties = new LinkedHashSet<>();
+ private SchemaStatus status = null;
+
+ /**
+ * Gets the name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Sets the name.
+ *
+ * @param name the new name
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Checks if is unique.
+ *
+ * @return true, if is unique
+ */
+ public boolean isUnique() {
+ return unique;
+ }
+
+ /**
+ * Sets the unique.
+ *
+ * @param unique the new unique
+ */
+ public void setUnique(boolean unique) {
+ this.unique = unique;
+ }
+
+ /**
+ * Gets the properties.
+ *
+ * @return the properties
+ */
+ public Set<DBProperty> getProperties() {
+ return properties;
+ }
+
+ /**
+ * Sets the properties.
+ *
+ * @param properties the new properties
+ */
+ public void setProperties(LinkedHashSet<DBProperty> properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * Gets the status.
+ *
+ * @return the status
+ */
+ public SchemaStatus getStatus() {
+ return status;
+ }
+
+ /**
+ * Sets the status.
+ *
+ * @param status the new status
+ */
+ public void setStatus(SchemaStatus status) {
+ this.status = status;
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/db/schema/DBProperty.java b/src/main/java/org/onap/aai/db/schema/DBProperty.java
new file mode 100644
index 0000000..491331d
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/DBProperty.java
@@ -0,0 +1,83 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import org.janusgraph.core.Cardinality;
+
+public class DBProperty implements Named {
+
+
+ private String name = null;
+ private Cardinality cardinality = null;
+ private Class<?> typeClass = null;
+
+ /**
+ * Gets the name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Sets the name.
+ *
+ * @param name the new name
+ */
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Gets the cardinality.
+ *
+ * @return the cardinality
+ */
+ public Cardinality getCardinality() {
+ return cardinality;
+ }
+
+ /**
+ * Sets the cardinality.
+ *
+ * @param cardinality the new cardinality
+ */
+ public void setCardinality(Cardinality cardinality) {
+ this.cardinality = cardinality;
+ }
+
+ /**
+ * Gets the type class.
+ *
+ * @return the type class
+ */
+ public Class<?> getTypeClass() {
+ return typeClass;
+ }
+
+ /**
+ * Sets the type class.
+ *
+ * @param type the new type class
+ */
+ public void setTypeClass(Class<?> type) {
+ this.typeClass = type;
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/db/schema/EdgeProperty.java b/src/main/java/org/onap/aai/db/schema/EdgeProperty.java
new file mode 100644
index 0000000..f89bc8f
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/EdgeProperty.java
@@ -0,0 +1,68 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.annotate.JsonPropertyOrder;
+import org.janusgraph.core.Multiplicity;
+
+@JsonPropertyOrder({ "label", "multiplicity" })
+public class EdgeProperty implements Named {
+
+ private String name = null;
+ private Multiplicity multiplicity = null;
+
+ /**
+ * Gets the name
+ */
+ @JsonProperty("label")
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Sets the name.
+ *
+ * @param name the new name
+ */
+ @JsonProperty("label")
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /**
+ * Gets the multiplicity.
+ *
+ * @return the multiplicity
+ */
+ public Multiplicity getMultiplicity() {
+ return multiplicity;
+ }
+
+ /**
+ * Sets the multiplicity.
+ *
+ * @param multiplicity the new multiplicity
+ */
+ public void setMultiplicity(Multiplicity multiplicity) {
+ this.multiplicity = multiplicity;
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/db/schema/ManageJanusGraphSchema.java b/src/main/java/org/onap/aai/db/schema/ManageJanusGraphSchema.java
new file mode 100644
index 0000000..dccc141
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/ManageJanusGraphSchema.java
@@ -0,0 +1,328 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphIndex;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.JanusGraphManagement.IndexBuilder;
+import org.janusgraph.core.schema.SchemaStatus;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.setup.SchemaVersion;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+public class ManageJanusGraphSchema {
+
+
+ private JanusGraphManagement graphMgmt;
+ private JanusGraph graph;
+ private List<DBProperty> aaiProperties;
+ private List<DBIndex> aaiIndexes;
+ private List<EdgeProperty> aaiEdgeProperties;
+ private Auditor oxmInfo = null;
+ private Auditor graphInfo = null;
+
+ /**
+ * Instantiates a new manage JanusGraph schema.
+ *
+ * @param graph the graph
+ */
+ public ManageJanusGraphSchema(final JanusGraph graph, AuditorFactory auditorFactory, SchemaVersions schemaVersions) {
+ this.graph = graph;
+ oxmInfo = auditorFactory.getOXMAuditor(schemaVersions.getDefaultVersion());
+ graphInfo = auditorFactory.getGraphAuditor(graph);
+ }
+
+
+ /**
+ * Builds the schema.
+ */
+ public void buildSchema() {
+
+ this.graphMgmt = graph.openManagement();
+ aaiProperties = new ArrayList<>();
+ aaiEdgeProperties = new ArrayList<>();
+ aaiIndexes = new ArrayList<>();
+ aaiProperties.addAll(oxmInfo.getAuditDoc().getProperties());
+ aaiIndexes.addAll(oxmInfo.getAuditDoc().getIndexes());
+ aaiEdgeProperties.addAll(oxmInfo.getAuditDoc().getEdgeLabels());
+ try {
+ createPropertyKeys();
+ createIndexes();
+ createEdgeLabels();
+ } catch (Exception e) {
+ e.printStackTrace();
+ graphMgmt.rollback();
+ }
+ graphMgmt.commit();
+ }
+
+ /**
+ * Creates the property keys.
+ */
+ private void createPropertyKeys() {
+
+
+ for (DBProperty prop : aaiProperties) {
+
+ if (graphMgmt.containsPropertyKey(prop.getName())) {
+ PropertyKey key = graphMgmt.getPropertyKey(prop.getName());
+ boolean isChanged = false;
+ if (!prop.getCardinality().equals(key.cardinality())) {
+ isChanged = true;
+ }
+ if (!prop.getTypeClass().equals(key.dataType())) {
+ isChanged = true;
+ }
+ if (isChanged) {
+ //must modify!
+ this.replaceProperty(prop);
+ }
+ } else {
+ //create a new property key
+ System.out.println("Key: " + prop.getName() + " not found - adding");
+ graphMgmt.makePropertyKey(prop.getName()).dataType(prop.getTypeClass()).cardinality(prop.getCardinality()).make();
+ }
+ }
+
+ }
+
+ /**
+ * Creates the indexes.
+ */
+ private void createIndexes() {
+
+ for (DBIndex index : aaiIndexes) {
+ Set<DBProperty> props = index.getProperties();
+ boolean isChanged = false;
+ boolean isNew = false;
+ List<PropertyKey> keyList = new ArrayList<>();
+ for (DBProperty prop : props) {
+ keyList.add(graphMgmt.getPropertyKey(prop.getName()));
+ }
+ if (graphMgmt.containsGraphIndex(index.getName())) {
+ JanusGraphIndex JanusGraphIndex = graphMgmt.getGraphIndex(index.getName());
+ PropertyKey[] dbKeys = JanusGraphIndex.getFieldKeys();
+ if (dbKeys.length != keyList.size()) {
+ isChanged = true;
+ } else {
+ int i = 0;
+ for (PropertyKey key : keyList) {
+ if (!dbKeys[i].equals(key)) {
+ isChanged = true;
+ break;
+ }
+ i++;
+ }
+ }
+ } else {
+ isNew = true;
+ }
+ if (keyList.size() > 0) {
+ this.createIndex(graphMgmt, index.getName(), keyList, index.isUnique(), isNew, isChanged);
+ }
+ }
+ }
+
+ // Use EdgeRules to make sure edgeLabels are defined in the db. NOTE: the multiplicty used here is
+ // always "MULTI". This is not the same as our internal "Many2Many", "One2One", "One2Many" or "Many2One"
+ // We use the same edge-label for edges between many different types of nodes and our internal
+ // multiplicty definitions depends on which two types of nodes are being connected.
+ /**
+ * Creates the edge labels.
+ */
+ private void createEdgeLabels() {
+
+
+ for (EdgeProperty prop : aaiEdgeProperties) {
+
+ if (graphMgmt.containsEdgeLabel(prop.getName())) {
+ // see what changed
+ } else {
+ graphMgmt.makeEdgeLabel(prop.getName()).multiplicity(prop.getMultiplicity()).make();
+ }
+
+ }
+
+
+ }
+
+ /**
+ * Creates the property.
+ *
+ * @param mgmt the mgmt
+ * @param prop the prop
+ */
+ private void createProperty(JanusGraphManagement mgmt, DBProperty prop) {
+ if (mgmt.containsPropertyKey(prop.getName())) {
+ PropertyKey key = mgmt.getPropertyKey(prop.getName());
+ boolean isChanged = false;
+ if (!prop.getCardinality().equals(key.cardinality())) {
+ isChanged = true;
+ }
+ if (!prop.getTypeClass().equals(key.dataType())) {
+ isChanged = true;
+ }
+ if (isChanged) {
+ //must modify!
+ this.replaceProperty(prop);
+ }
+ } else {
+ //create a new property key
+ System.out.println("Key: " + prop.getName() + " not found - adding");
+ mgmt.makePropertyKey(prop.getName()).dataType(prop.getTypeClass()).cardinality(prop.getCardinality()).make();
+ }
+ }
+
+ /**
+ * Creates the index.
+ *
+ * @param mgmt the mgmt
+ * @param indexName the index name
+ * @param keys the keys
+ * @param isUnique the is unique
+ * @param isNew the is new
+ * @param isChanged the is changed
+ */
+ private void createIndex(JanusGraphManagement mgmt, String indexName, List<PropertyKey> keys, boolean isUnique, boolean isNew, boolean isChanged) {
+
+ /*if (isChanged) {
+ System.out.println("Changing index: " + indexName);
+ JanusGraphIndex oldIndex = mgmt.getGraphIndex(indexName);
+ mgmt.updateIndex(oldIndex, SchemaAction.DISABLE_INDEX);
+ mgmt.commit();
+ //cannot remove indexes
+ //graphMgmt.updateIndex(oldIndex, SchemaAction.REMOVE_INDEX);
+ }*/
+ if (isNew || isChanged) {
+
+ if (isNew) {
+ IndexBuilder builder = mgmt.buildIndex(indexName,Vertex.class);
+ for (PropertyKey k : keys) {
+ builder.addKey(k);
+ }
+ if (isUnique) {
+ builder.unique();
+ }
+ builder.buildCompositeIndex();
+ System.out.println("Built index for " + indexName + " with keys: " + keys);
+
+ //mgmt.commit();
+ }
+
+ //mgmt = graph.asAdmin().getManagementSystem();
+ //mgmt.updateIndex(mgmt.getGraphIndex(indexName), SchemaAction.REGISTER_INDEX);
+ //mgmt.commit();
+
+ try {
+ //waitForCompletion(indexName);
+ //JanusGraphIndexRepair.hbaseRepair(AAIConstants.AAI_CONFIG_FILENAME, indexName, "");
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ graph.tx().rollback();
+ graph.close();
+ e.printStackTrace();
+ }
+
+ //mgmt = graph.asAdmin().getManagementSystem();
+ //mgmt.updateIndex(mgmt.getGraphIndex(indexName), SchemaAction.REINDEX);
+
+ //mgmt.updateIndex(mgmt.getGraphIndex(indexName), SchemaAction.ENABLE_INDEX);
+
+ //mgmt.commit();
+
+ }
+ }
+
+ /**
+ * Wait for completion.
+ *
+ * @param name the name
+ * @throws InterruptedException the interrupted exception
+ */
+ private void waitForCompletion(String name) throws InterruptedException {
+
+ boolean registered = false;
+ long before = System.currentTimeMillis();
+ while (!registered) {
+ Thread.sleep(500L);
+ JanusGraphManagement mgmt = graph.openManagement();
+ JanusGraphIndex idx = mgmt.getGraphIndex(name);
+ registered = true;
+ for (PropertyKey k : idx.getFieldKeys()) {
+ SchemaStatus s = idx.getIndexStatus(k);
+ registered &= s.equals(SchemaStatus.REGISTERED);
+ }
+ mgmt.rollback();
+ }
+ System.out.println("Index REGISTERED in " + (System.currentTimeMillis() - before) + " ms");
+ }
+
+ /**
+ * Replace property.
+ *
+ * @param key the key
+ */
+ private void replaceProperty(DBProperty key) {
+
+
+
+
+ }
+
+ /**
+ * Update index.
+ *
+ * @param index the index
+ */
+ public void updateIndex(DBIndex index) {
+
+ JanusGraphManagement mgmt = graph.openManagement();
+ List<PropertyKey> keys = new ArrayList<>();
+ boolean isNew = false;
+ boolean isChanged = false;
+ for (DBProperty prop : index.getProperties()) {
+ createProperty(mgmt, prop);
+ keys.add(mgmt.getPropertyKey(prop.getName()));
+ }
+ if (mgmt.containsGraphIndex(index.getName())) {
+ System.out.println("index already exists");
+ isNew = false;
+ isChanged = true;
+ } else {
+ isNew = true;
+ isChanged = false;
+ }
+ this.createIndex(mgmt, index.getName(), keys, index.isUnique(), isNew, isChanged);
+
+ mgmt.commit();
+
+ }
+
+
+
+
+
+}
diff --git a/src/main/java/org/onap/aai/db/schema/Named.java b/src/main/java/org/onap/aai/db/schema/Named.java
new file mode 100644
index 0000000..f12699b
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/Named.java
@@ -0,0 +1,30 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+public interface Named {
+
+ /**
+ * Gets the name.
+ *
+ * @return the name
+ */
+ public String getName();
+}
diff --git a/src/main/java/org/onap/aai/db/schema/ScriptDriver.java b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java
new file mode 100644
index 0000000..dca8e83
--- /dev/null
+++ b/src/main/java/org/onap/aai/db/schema/ScriptDriver.java
@@ -0,0 +1,123 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.db.schema;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.codehaus.jackson.JsonGenerationException;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.util.AAIConfig;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraph;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+public class ScriptDriver {
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ * @throws AAIException the AAI exception
+ * @throws JsonGenerationException the json generation exception
+ * @throws JsonMappingException the json mapping exception
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ public static void main (String[] args) throws AAIException, IOException, ConfigurationException {
+ CommandLineArgs cArgs = new CommandLineArgs();
+
+ LoggingContext.init();
+ LoggingContext.component("DBSchemaScriptDriver");
+ LoggingContext.partnerName("NA");
+ LoggingContext.targetEntity("AAI");
+ LoggingContext.requestId(UUID.randomUUID().toString());
+ LoggingContext.serviceName("AAI");
+ LoggingContext.targetServiceName("main");
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ new JCommander(cArgs, args);
+
+ if (cArgs.help) {
+ System.out.println("-c [path to graph configuration] -type [what you want to audit - oxm or graph]");
+ }
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ AuditorFactory auditorFactory = ctx.getBean(AuditorFactory.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+
+ String config = cArgs.config;
+ AAIConfig.init();
+
+ PropertiesConfiguration graphConfiguration = new AAIGraphConfig
+ .Builder(config)
+ .forService(ScriptDriver.class.getSimpleName())
+ .withGraphType("NA")
+ .buildConfiguration();
+
+ try (JanusGraph graph = JanusGraphFactory.open(graphConfiguration)) {
+ if (!("oxm".equals(cArgs.type) || "graph".equals(cArgs.type))) {
+ System.out.println("type: " + cArgs.type + " not recognized.");
+ System.exit(1);
+ }
+
+ AuditDoc doc = null;
+ if ("oxm".equals(cArgs.type)) {
+ doc = auditorFactory.getOXMAuditor(schemaVersions.getDefaultVersion()).getAuditDoc();
+ } else if ("graph".equals(cArgs.type)) {
+ doc = auditorFactory.getGraphAuditor(graph).getAuditDoc();
+ }
+
+ ObjectMapper mapper = new ObjectMapper();
+
+ String json = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(doc);
+ System.out.println(json);
+ }
+ }
+
+}
+
+class CommandLineArgs {
+
+ @Parameter(names = "--help", description = "Help")
+ public boolean help = false;
+
+ @Parameter(names = "-c", description = "Configuration", required=true)
+ public String config;
+
+ @Parameter(names = "-type", description = "Type", required=true)
+ public String type = "graph";
+
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/dbgen/DupeTool.java b/src/main/java/org/onap/aai/dbgen/DupeTool.java
new file mode 100644
index 0000000..7b7ef99
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/DupeTool.java
@@ -0,0 +1,1854 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.*;
+import java.util.Map.Entry;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.edges.enums.EdgeProperty;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraph;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+public class DupeTool {
+
+ private static final EELFLogger logger = EELFManager.getInstance().getLogger(DupeTool.class.getSimpleName());
+ private static final String FROMAPPID = "AAI-DB";
+ private static final String TRANSID = UUID.randomUUID().toString();
+
+ private static String graphType = "realdb";
+ private final SchemaVersions schemaVersions;
+
+ private boolean shouldExitVm = true;
+
+ public void exit(int statusCode) {
+ if (this.shouldExitVm) {
+ System.exit(1);
+ }
+ }
+
+ private LoaderFactory loaderFactory;
+
+ public DupeTool(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
+ this(loaderFactory, schemaVersions, true);
+ }
+
+ public DupeTool(LoaderFactory loaderFactory, SchemaVersions schemaVersions, boolean shouldExitVm){
+ this.loaderFactory = loaderFactory;
+ this.schemaVersions = schemaVersions;
+ this.shouldExitVm = shouldExitVm;
+ }
+
+ public void execute(String[] args){
+
+ String defVersion = "v12";
+ try {
+ defVersion = AAIConfig.get(AAIConstants.AAI_DEFAULT_API_VERSION_PROP);
+ } catch (AAIException ae) {
+ String emsg = "Error trying to get default API Version property \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+
+
+ Loader loader = null;
+ try {
+ loader = loaderFactory.createLoaderForVersion(ModelType.MOXY, schemaVersions.getDefaultVersion());
+ } catch (Exception ex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logger.error("ERROR - Could not do the moxyMod.init() " + LogFormatTools.getStackTop(ex));
+ exit(1);
+ }
+ JanusGraph graph1 = null;
+ JanusGraph graph2 = null;
+ Graph gt1 = null;
+ Graph gt2 = null;
+
+ boolean specialTenantRule = false;
+
+ try {
+ AAIConfig.init();
+ int maxRecordsToFix = AAIConstants.AAI_DUPETOOL_DEFAULT_MAX_FIX;
+ int sleepMinutes = AAIConstants.AAI_DUPETOOL_DEFAULT_SLEEP_MINUTES;
+ int timeWindowMinutes = 0; // A value of 0 means that we will not have a time-window -- we will look
+ // at all nodes of the passed-in nodeType.
+ long windowStartTime = 0; // Translation of the window into a starting timestamp
+
+ try {
+ String maxFixStr = AAIConfig.get("aai.dupeTool.default.max.fix");
+ if (maxFixStr != null && !maxFixStr.equals("")) {
+ maxRecordsToFix = Integer.parseInt(maxFixStr);
+ }
+ String sleepStr = AAIConfig.get("aai.dupeTool.default.sleep.minutes");
+ if (sleepStr != null && !sleepStr.equals("")) {
+ sleepMinutes = Integer.parseInt(sleepStr);
+ }
+ } catch (Exception e) {
+ // Don't worry, we'll just use the defaults that we got from AAIConstants
+ logger.warn("WARNING - could not pick up aai.dupeTool values from aaiconfig.properties file. Will use defaults. ");
+ }
+
+ String nodeTypeVal = "";
+ String userIdVal = "";
+ String filterParams = "";
+ Boolean skipHostCheck = false;
+ Boolean autoFix = false;
+ String argStr4Msg = "";
+ Introspector obj = null;
+
+ if (args != null && args.length > 0) {
+ // They passed some arguments in that will affect processing
+ for (int i = 0; i < args.length; i++) {
+ String thisArg = args[i];
+ argStr4Msg = argStr4Msg + " " + thisArg;
+
+ if (thisArg.equals("-nodeType")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -nodeType option. ");
+ exit(0);
+ }
+ nodeTypeVal = args[i];
+ argStr4Msg = argStr4Msg + " " + nodeTypeVal;
+ } else if (thisArg.equals("-sleepMinutes")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error("No value passed with -sleepMinutes option.");
+ exit(0);
+ }
+ String nextArg = args[i];
+ try {
+ sleepMinutes = Integer.parseInt(nextArg);
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error("Bad value passed with -sleepMinutes option: ["
+ + nextArg + "]");
+ exit(0);
+ }
+ argStr4Msg = argStr4Msg + " " + sleepMinutes;
+ } else if (thisArg.equals("-maxFix")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error("No value passed with -maxFix option.");
+ exit(0);
+ }
+ String nextArg = args[i];
+ try {
+ maxRecordsToFix = Integer.parseInt(nextArg);
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error("Bad value passed with -maxFix option: ["
+ + nextArg + "]");
+ exit(0);
+ }
+ argStr4Msg = argStr4Msg + " " + maxRecordsToFix;
+ } else if (thisArg.equals("-timeWindowMinutes")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error("No value passed with -timeWindowMinutes option.");
+ exit(0);
+ }
+ String nextArg = args[i];
+ try {
+ timeWindowMinutes = Integer.parseInt(nextArg);
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error("Bad value passed with -timeWindowMinutes option: ["
+ + nextArg + "]");
+ exit(0);
+ }
+ argStr4Msg = argStr4Msg + " " + timeWindowMinutes;
+ } else if (thisArg.equals("-skipHostCheck")) {
+ skipHostCheck = true;
+ } else if (thisArg.equals("-specialTenantRule")) {
+ specialTenantRule = true;
+ } else if (thisArg.equals("-autoFix")) {
+ autoFix = true;
+ } else if (thisArg.equals("-userId")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -userId option. ");
+ exit(0);
+ }
+ userIdVal = args[i];
+ argStr4Msg = argStr4Msg + " " + userIdVal;
+ } else if (thisArg.equals("-params4Collect")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -params4Collect option. ");
+ exit(0);
+ }
+ filterParams = args[i];
+ argStr4Msg = argStr4Msg + " " + filterParams;
+ } else {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" Unrecognized argument passed to DupeTool: ["
+ + thisArg + "]. ");
+ logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection ");
+ exit(0);
+ }
+ }
+ }
+
+ userIdVal = userIdVal.trim();
+ if ((userIdVal.length() < 6) || userIdVal.toUpperCase().equals("AAIADMIN")) {
+ String emsg = "userId parameter is required. [" + userIdVal + "] passed to DupeTool(). userId must be not empty and not aaiadmin \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+
+ nodeTypeVal = nodeTypeVal.trim();
+ if (nodeTypeVal.equals("")) {
+ String emsg = " nodeType is a required parameter for DupeTool().\n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(emsg);
+ exit(0);
+ } else {
+ obj = loader.introspectorFromName(nodeTypeVal);
+ }
+
+ if (timeWindowMinutes > 0) {
+ // Translate the window value (ie. 30 minutes) into a unix timestamp like
+ // we use in the db - so we can select data created after that time.
+ windowStartTime = figureWindowStartTime(timeWindowMinutes);
+ }
+
+ String msg = "";
+ msg = "DupeTool called with these params: [" + argStr4Msg + "]";
+ System.out.println(msg);
+ logger.info(msg);
+
+ // Determine what the key fields are for this nodeType (and we want them ordered)
+ ArrayList<String> keyPropNamesArr = new ArrayList<String>(obj.getKeys());
+
+ // Determine what kinds of nodes (if any) this nodeType is dependent on for uniqueness
+ ArrayList<String> depNodeTypeList = new ArrayList<String>();
+ Collection<String> depNTColl = obj.getDependentOn();
+ Iterator<String> ntItr = depNTColl.iterator();
+ while (ntItr.hasNext()) {
+ depNodeTypeList.add(ntItr.next());
+ }
+
+ // Based on the nodeType, window and filterData, figure out the vertices that we will be checking
+ System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n");
+ graph1 = setupGraph(logger);
+ gt1 = getGraphTransaction(graph1, logger);
+ ArrayList<Vertex> verts2Check = new ArrayList<Vertex>();
+ try {
+ verts2Check = figureOutNodes2Check(TRANSID, FROMAPPID, gt1,
+ nodeTypeVal, windowStartTime, filterParams, logger);
+ } catch (AAIException ae) {
+ String emsg = "Error trying to get initial set of nodes to check. \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+
+ if (verts2Check == null || verts2Check.size() == 0) {
+ msg = " No vertices found to check. Used nodeType = [" + nodeTypeVal
+ + "], windowMinutes = " + timeWindowMinutes
+ + ", filterData = [" + filterParams + "].";
+ logger.info(msg);
+ System.out.println(msg);
+ exit(0);
+ } else {
+ msg = " Found " + verts2Check.size() + " nodes of type " + nodeTypeVal
+ + " to check using passed filterParams and windowStartTime. ";
+ logger.info(msg);
+ System.out.println(msg);
+ }
+
+ ArrayList<String> firstPassDupeSets = new ArrayList<String>();
+ ArrayList<String> secondPassDupeSets = new ArrayList<String>();
+ Boolean isDependentOnParent = false;
+ if (!obj.getDependentOn().isEmpty()) {
+ isDependentOnParent = true;
+ }
+
+ if (isDependentOnParent) {
+ firstPassDupeSets = getDupeSets4DependentNodes(TRANSID, FROMAPPID, gt1,
+ defVersion, nodeTypeVal, verts2Check, keyPropNamesArr, loader,
+ specialTenantRule, logger);
+ } else {
+ firstPassDupeSets = getDupeSets4NonDepNodes(TRANSID, FROMAPPID, gt1,
+ defVersion, nodeTypeVal, verts2Check, keyPropNamesArr,
+ specialTenantRule, loader, logger);
+ }
+
+ msg = " Found " + firstPassDupeSets.size() + " sets of duplicates for this request. ";
+ logger.info(msg);
+ System.out.println(msg);
+ if (firstPassDupeSets.size() > 0) {
+ msg = " Here is what they look like: ";
+ logger.info(msg);
+ System.out.println(msg);
+ for (int x = 0; x < firstPassDupeSets.size(); x++) {
+ msg = " Set " + x + ": [" + firstPassDupeSets.get(x) + "] ";
+ logger.info(msg);
+ System.out.println(msg);
+ showNodeDetailsForADupeSet(gt1, firstPassDupeSets.get(x), logger);
+ }
+ }
+
+ boolean didSomeDeletesFlag = false;
+ ArrayList<String> dupeSetsToFix = new ArrayList<String>();
+ if (autoFix && firstPassDupeSets.size() == 0) {
+ msg = "AutoFix option is on, but no dupes were found on the first pass. Nothing to fix.";
+ logger.info(msg);
+ System.out.println(msg);
+ } else if (autoFix) {
+ // We will try to fix any dupes that we can - but only after sleeping for a
+ // time and re-checking the list of duplicates using a seperate transaction.
+ try {
+ msg = "\n\n----------- About to sleep for " + sleepMinutes + " minutes."
+ + " -----------\n\n";
+ logger.info(msg);
+ System.out.println(msg);
+ int sleepMsec = sleepMinutes * 60 * 1000;
+ Thread.sleep(sleepMsec);
+ } catch (InterruptedException ie) {
+ msg = "\n >>> Sleep Thread has been Interrupted <<< ";
+ logger.info(msg);
+ System.out.println(msg);
+ exit(0);
+ }
+
+ graph2 = setupGraph(logger);
+ gt2 = getGraphTransaction(graph2, logger);
+ if (isDependentOnParent) {
+ secondPassDupeSets = getDupeSets4DependentNodes(TRANSID, FROMAPPID, gt2,
+ defVersion, nodeTypeVal, verts2Check, keyPropNamesArr, loader,
+ specialTenantRule, logger);
+ } else {
+ secondPassDupeSets = getDupeSets4NonDepNodes(TRANSID, FROMAPPID, gt2,
+ defVersion, nodeTypeVal, verts2Check, keyPropNamesArr,
+ specialTenantRule, loader, logger);
+ }
+
+ dupeSetsToFix = figureWhichDupesStillNeedFixing(firstPassDupeSets, secondPassDupeSets, logger);
+ msg = "\nAfter running a second pass, there were " + dupeSetsToFix.size()
+ + " sets of duplicates that we think can be deleted. ";
+ logger.info(msg);
+ System.out.println(msg);
+ if (dupeSetsToFix.size() > 0) {
+ msg = " Here is what the sets look like: ";
+ logger.info(msg);
+ System.out.println(msg);
+ for (int x = 0; x < dupeSetsToFix.size(); x++) {
+ msg = " Set " + x + ": [" + dupeSetsToFix.get(x) + "] ";
+ logger.info(msg);
+ System.out.println(msg);
+ showNodeDetailsForADupeSet(gt2, dupeSetsToFix.get(x), logger);
+ }
+ }
+
+ if (dupeSetsToFix.size() > 0) {
+ if (dupeSetsToFix.size() > maxRecordsToFix) {
+ String infMsg = " >> WARNING >> Dupe list size ("
+ + dupeSetsToFix.size()
+ + ") is too big. The maxFix we are using is: "
+ + maxRecordsToFix
+ + ". No nodes will be deleted. (use the"
+ + " -maxFix option to override this limit.)";
+ System.out.println(infMsg);
+ logger.info(infMsg);
+ } else {
+ // Call the routine that fixes known dupes
+ didSomeDeletesFlag = deleteNonKeepers(gt2, dupeSetsToFix, logger);
+ }
+ }
+ if (didSomeDeletesFlag) {
+ gt2.tx().commit();
+ }
+ }
+
+ } catch (AAIException e) {
+ logger.error("Caught AAIException while running the dupeTool: " + LogFormatTools.getStackTop(e));
+ ErrorLogHelper.logException(e);
+ } catch (Exception ex) {
+ logger.error("Caught exception while running the dupeTool: " + LogFormatTools.getStackTop(ex));
+ ErrorLogHelper.logError("AAI_6128", ex.getMessage() + ", resolve and rerun the dupeTool. ");
+ } finally {
+ if (gt1 != null && gt1.tx().isOpen()) {
+ // We don't change any data with gt1 - so just roll it back so it knows we're done.
+ try {
+ gt1.tx().rollback();
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed
+ logger.warn("WARNING from final gt1.rollback() " + LogFormatTools.getStackTop(ex));
+ }
+ }
+
+ if (gt2 != null && gt2.tx().isOpen()) {
+ // Any changes that worked correctly should have already done
+ // their commits.
+ try {
+ gt2.tx().rollback();
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed
+ logger.warn("WARNING from final gt2.rollback() " + LogFormatTools.getStackTop(ex));
+ }
+ }
+
+ try {
+ if (graph1 != null && graph1.isOpen()) {
+ closeGraph(graph1, logger);
+ }
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
+ logger.warn("WARNING from final graph1.shutdown() " + LogFormatTools.getStackTop(ex));
+ }
+
+ try {
+ if (graph2 != null && graph2.isOpen()) {
+ closeGraph(graph2, logger);
+ }
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
+ logger.warn("WARNING from final graph2.shutdown() " + LogFormatTools.getStackTop(ex));
+ }
+ }
+
+ exit(0);
+ }
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args) {
+
+ System.setProperty("aai.service.name", DupeTool.class.getSimpleName());
+ // Set the logging file properties to be used by EELFManager
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "dupeTool-logback.xml");
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ MDC.put("logFilenameAppender", DupeTool.class.getSimpleName());
+
+ LoggingContext.init();
+ LoggingContext.partnerName(FROMAPPID);
+ LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.component("dupeTool");
+ LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.targetServiceName("main");
+ LoggingContext.requestId(TRANSID);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+ DupeTool dupeTool = new DupeTool(loaderFactory, schemaVersions);
+ dupeTool.execute(args);
+ }// end of main()
+
+
+ /**
+ * Collect Duplicate Sets for nodes that are NOT dependent on parent nodes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param version the version
+ * @param nType the n type
+ * @param passedVertList the passed vert list
+ * @param dbMaps the db maps
+ * @return the array list
+ */
+ private ArrayList<String> getDupeSets4NonDepNodes(String transId,
+ String fromAppId, Graph g, String version, String nType,
+ ArrayList<Vertex> passedVertList,
+ ArrayList<String> keyPropNamesArr,
+ Boolean specialTenantRule, Loader loader, EELFLogger logger) {
+
+ ArrayList<String> returnList = new ArrayList<String>();
+
+ // We've been passed a set of nodes that we want to check.
+ // They are all NON-DEPENDENT nodes meaning that they should be
+ // unique in the DB based on their KEY DATA alone. So, if
+ // we group them by their key data - if any key has more than one
+ // vertex mapped to it, those vertices are dupes.
+ //
+ // When we find duplicates, we return then as a String (there can be
+ // more than one duplicate for one set of key data):
+ // Each element in the returned arrayList might look like this:
+ // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
+ // couldn't figure out which one to keep)
+ // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
+ // thought the third one was the one that should survive)
+
+ HashMap<String, ArrayList<String>> keyVals2VidHash = new HashMap<String, ArrayList<String>>();
+ HashMap<String, Vertex> vtxHash = new HashMap<String, Vertex>();
+ Iterator<Vertex> pItr = passedVertList.iterator();
+ while (pItr.hasNext()) {
+ try {
+ Vertex tvx = pItr.next();
+ String thisVid = tvx.id().toString();
+ vtxHash.put(thisVid, tvx);
+
+ // if there are more than one vertexId mapping to the same keyProps -- they are dupes
+ String hKey = getNodeKeyValString(tvx, keyPropNamesArr, logger);
+ if (keyVals2VidHash.containsKey(hKey)) {
+ // We've already seen this key
+ ArrayList<String> tmpVL = (ArrayList<String>) keyVals2VidHash.get(hKey);
+ tmpVL.add(thisVid);
+ keyVals2VidHash.put(hKey, tmpVL);
+ } else {
+ // First time for this key
+ ArrayList<String> tmpVL = new ArrayList<String>();
+ tmpVL.add(thisVid);
+ keyVals2VidHash.put(hKey, tmpVL);
+ }
+ } catch (Exception e) {
+ logger.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. " + LogFormatTools.getStackTop(e));
+ }
+ }
+
+ for (Map.Entry<String, ArrayList<String>> entry : keyVals2VidHash.entrySet()) {
+ ArrayList<String> vidList = entry.getValue();
+ try {
+ if (!vidList.isEmpty() && vidList.size() > 1) {
+ // There are more than one vertex id's using the same key info
+ String dupesStr = "";
+ ArrayList<Vertex> vertList = new ArrayList<Vertex>();
+ for (int i = 0; i < vidList.size(); i++) {
+ String tmpVid = vidList.get(i);
+ dupesStr = dupesStr + tmpVid + "|";
+ vertList.add(vtxHash.get(tmpVid));
+ }
+
+ if (dupesStr != "") {
+ Vertex prefV = getPreferredDupe(transId, fromAppId,
+ g, vertList, version, specialTenantRule, loader, logger);
+ if (prefV == null) {
+ // We could not determine which duplicate to keep
+ dupesStr = dupesStr + "KeepVid=UNDETERMINED";
+ returnList.add(dupesStr);
+ } else {
+ dupesStr = dupesStr + "KeepVid=" + prefV.id();
+ returnList.add(dupesStr);
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.warn(" >>> Threw an error in getDupeSets4NonDepNodes - just absorb this error and move on. " + LogFormatTools.getStackTop(e));
+ }
+
+ }
+ return returnList;
+
+ }// End of getDupeSets4NonDepNodes()
+
+
+ /**
+ * Collect Duplicate Sets for nodes that are dependent on parent nodes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param version the version
+ * @param nType the n type
+ * @param passedVertList the passed vert list
+ * @param dbMaps the db maps
+ * @param keyPropNamesArr Array (ordered) of keyProperty names
+ * @param specialTenantRule flag
+ * @param EELFLogger the logger
+ * @return the array list
+ */
+ private ArrayList<String> getDupeSets4DependentNodes(String transId,
+ String fromAppId, Graph g, String version, String nType,
+ ArrayList<Vertex> passedVertList,
+ ArrayList<String> keyPropNamesArr, Loader loader,
+ Boolean specialTenantRule, EELFLogger logger) {
+
+ // This is for nodeTypes that DEPEND ON A PARENT NODE FOR UNIQUNESS
+
+ ArrayList<String> returnList = new ArrayList<String>();
+ ArrayList<String> alreadyFoundDupeVidArr = new ArrayList<String>();
+
+ // We've been passed a set of nodes that we want to check. These are
+ // all nodes that ARE DEPENDENT on a PARENT Node for uniqueness.
+ // The first thing to do is to identify the key properties for the node-type
+ // and pull from the db just using those properties.
+ // Then, we'll check those nodes with their parent nodes to see if there
+ // are any duplicates.
+ //
+ // When we find duplicates, we return then as a String (there can be
+ // more than one duplicate for one set of key data):
+ // Each element in the returned arrayList might look like this:
+ // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
+ // couldn't figure out which one to keep)
+ // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
+ // thought the third one was the one that should survive)
+ HashMap<String, Object> checkVertHash = new HashMap<String, Object>();
+ try {
+ Iterator<Vertex> pItr = passedVertList.iterator();
+ while (pItr.hasNext()) {
+ Vertex tvx = pItr.next();
+ String passedId = tvx.id().toString();
+ if (!alreadyFoundDupeVidArr.contains(passedId)) {
+ // We haven't seen this one before - so we should check it.
+ HashMap<String, Object> keyPropValsHash = getNodeKeyVals(tvx, keyPropNamesArr, logger);
+ ArrayList<Vertex> tmpVertList = getNodeJustUsingKeyParams(transId, fromAppId, g,
+ nType, keyPropValsHash, version, logger);
+
+ if (tmpVertList.size() <= 1) {
+ // Even without a parent node, this thing is unique so don't worry about it.
+ } else {
+ for (int i = 0; i < tmpVertList.size(); i++) {
+ Vertex tmpVtx = (tmpVertList.get(i));
+ String tmpVid = tmpVtx.id().toString();
+ alreadyFoundDupeVidArr.add(tmpVid);
+
+ String hKey = getNodeKeyValString(tmpVtx, keyPropNamesArr, logger);
+ if (checkVertHash.containsKey(hKey)) {
+ // add it to an existing list
+ ArrayList<Vertex> tmpVL = (ArrayList<Vertex>) checkVertHash.get(hKey);
+ tmpVL.add(tmpVtx);
+ checkVertHash.put(hKey, tmpVL);
+ } else {
+ // First time for this key
+ ArrayList<Vertex> tmpVL = new ArrayList<Vertex>();
+ tmpVL.add(tmpVtx);
+ checkVertHash.put(hKey, tmpVL);
+ }
+ }
+ }
+ }
+ }
+
+ // More than one node have the same key fields since they may
+ // depend on a parent node for uniqueness. Since we're finding
+ // more than one, we want to check to see if any of the
+ // vertices that have this set of keys are also pointing at the
+ // same 'parent' node.
+ // Note: for a given set of key data, it is possible that there
+ // could be more than one set of duplicates.
+ for (Entry<String, Object> lentry : checkVertHash.entrySet()) {
+ ArrayList<Vertex> thisIdSetList = (ArrayList<Vertex>) lentry.getValue();
+ if (thisIdSetList == null || thisIdSetList.size() < 2) {
+ // Nothing to check for this set.
+ continue;
+ }
+
+ HashMap<String, ArrayList<Vertex>> vertsGroupedByParentHash = groupVertsByDepNodes(
+ transId, fromAppId, g, version, nType,
+ thisIdSetList, loader);
+ for (Map.Entry<String, ArrayList<Vertex>> entry : vertsGroupedByParentHash
+ .entrySet()) {
+ ArrayList<Vertex> thisParentsVertList = entry
+ .getValue();
+ if (thisParentsVertList.size() > 1) {
+ // More than one vertex found with the same key info
+ // hanging off the same parent/dependent node
+ String dupesStr = "";
+ for (int i = 0; i < thisParentsVertList.size(); i++) {
+ dupesStr = dupesStr
+ + ((thisParentsVertList
+ .get(i))).id() + "|";
+ }
+ if (dupesStr != "") {
+ Vertex prefV = getPreferredDupe(transId,
+ fromAppId, g, thisParentsVertList,
+ version, specialTenantRule, loader, logger);
+
+ if (prefV == null) {
+ // We could not determine which duplicate to keep
+ dupesStr = dupesStr + "KeepVid=UNDETERMINED";
+ returnList.add(dupesStr);
+ } else {
+ dupesStr = dupesStr + "KeepVid="
+ + prefV.id().toString();
+ returnList.add(dupesStr);
+ }
+ }
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ logger.warn(" >>> Threw an error in checkAndProcessDupes - just absorb this error and move on. " + LogFormatTools.getStackTop(e));
+ }
+
+ return returnList;
+
+ }// End of getDupeSets4DependentNodes()
+
+
+ private Graph getGraphTransaction(JanusGraph graph, EELFLogger logger) {
+
+ Graph gt = null;
+ try {
+ if (graph == null) {
+ String emsg = "could not get graph object in DupeTool. \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+ gt = graph.newTransaction();
+ if (gt == null) {
+ String emsg = "null graphTransaction object in DupeTool. \n";
+ throw new AAIException("AAI_6101", emsg);
+ }
+
+ } catch (AAIException e1) {
+ String msg = e1.getErrorObject().toString();
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(msg);
+ exit(0);
+ } catch (Exception e2) {
+ String msg = e2.toString();
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logger.error(msg);
+ exit(0);
+ }
+
+ return gt;
+
+ }// End of getGraphTransaction()
+
+
+ public void showNodeInfo(EELFLogger logger, Vertex tVert, Boolean displayAllVidsFlag) {
+
+ try {
+ Iterator<VertexProperty<Object>> pI = tVert.properties();
+ String infStr = ">>> Found Vertex with VertexId = " + tVert.id() + ", properties: ";
+ System.out.println(infStr);
+ logger.info(infStr);
+ while (pI.hasNext()) {
+ VertexProperty<Object> tp = pI.next();
+ infStr = " [" + tp.key() + "|" + tp.value() + "] ";
+ System.out.println(infStr);
+ logger.info(infStr);
+ }
+
+ ArrayList<String> retArr = collectEdgeInfoForNode(logger, tVert, displayAllVidsFlag);
+ for (String infoStr : retArr) {
+ System.out.println(infoStr);
+ logger.info(infoStr);
+ }
+ } catch (Exception e) {
+ String warnMsg = " -- Error -- trying to display edge info. [" + e.getMessage() + "]";
+ System.out.println(warnMsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logger.warn(warnMsg);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+ }
+
+ }// End of showNodeInfo()
+
+
+ public ArrayList<String> collectEdgeInfoForNode(EELFLogger logger, Vertex tVert, boolean displayAllVidsFlag) {
+ ArrayList<String> retArr = new ArrayList<String>();
+ Direction dir = Direction.OUT;
+ for (int i = 0; i <= 1; i++) {
+ if (i == 1) {
+ // Second time through we'll look at the IN edges.
+ dir = Direction.IN;
+ }
+ Iterator<Edge> eI = tVert.edges(dir);
+ if (!eI.hasNext()) {
+ retArr.add("No " + dir + " edges were found for this vertex. ");
+ }
+ while (eI.hasNext()) {
+ Edge ed = eI.next();
+ String lab = ed.label();
+ Vertex vtx = null;
+ if (dir == Direction.OUT) {
+ // get the vtx on the "other" side
+ vtx = ed.inVertex();
+ } else {
+ // get the vtx on the "other" side
+ vtx = ed.outVertex();
+ }
+ if (vtx == null) {
+ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ } else {
+ String nType = vtx.<String>property("aai-node-type").orElse(null);
+ if (displayAllVidsFlag) {
+ // This should rarely be needed
+ String vid = vtx.id().toString();
+ retArr.add("Found an " + dir + " edge (" + lab + ") between this vertex and a [" + nType + "] node with VtxId = " + vid);
+ } else {
+ // This is the normal case
+ retArr.add("Found an " + dir + " edge (" + lab + ") between this vertex and a [" + nType + "] node. ");
+ }
+ }
+ }
+ }
+ return retArr;
+
+ }// end of collectEdgeInfoForNode()
+
+
+ private long figureWindowStartTime(int timeWindowMinutes) {
+ // Given a window size, calculate what the start-timestamp would be.
+
+ if (timeWindowMinutes <= 0) {
+ // This just means that there is no window...
+ return 0;
+ }
+ long unixTimeNow = System.currentTimeMillis();
+ long windowInMillis = timeWindowMinutes * 60 * 1000;
+
+ long startTimeStamp = unixTimeNow - windowInMillis;
+
+ return startTimeStamp;
+ } // End of figureWindowStartTime()
+
+
+ /**
+ * Gets the node(s) just using key params.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param graph the graph
+ * @param nodeType the node type
+ * @param keyPropsHash the key props hash
+ * @param apiVersion the api version
+ * @return the node just using key params
+ * @throws AAIException the AAI exception
+ */
+ public ArrayList<Vertex> getNodeJustUsingKeyParams(String transId, String fromAppId, Graph graph, String nodeType,
+ HashMap<String, Object> keyPropsHash, String apiVersion, EELFLogger logger) throws AAIException {
+
+ ArrayList<Vertex> retVertList = new ArrayList<Vertex>();
+
+ // We assume that all NodeTypes have at least one key-property defined.
+ // Note - instead of key-properties (the primary key properties), a user could pass
+ // alternate-key values if they are defined for the nodeType.
+ ArrayList<String> kName = new ArrayList<String>();
+ ArrayList<Object> kVal = new ArrayList<Object>();
+ if (keyPropsHash == null || keyPropsHash.isEmpty()) {
+ throw new AAIException("AAI_6120", " NO key properties passed for this getNodeJustUsingKeyParams() request. NodeType = [" + nodeType + "]. ");
+ }
+
+ int i = -1;
+ for (Map.Entry<String, Object> entry : keyPropsHash.entrySet()) {
+ i++;
+ kName.add(i, entry.getKey());
+ kVal.add(i, entry.getValue());
+ }
+ int topPropIndex = i;
+ Vertex tiV = null;
+ String propsAndValuesForMsg = "";
+ Iterator<Vertex> verts = null;
+ GraphTraversalSource g = graph.traversal();
+ try {
+ if (topPropIndex == 0) {
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ") ";
+ verts = g.V().has(kName.get(0), kVal.get(0)).has("aai-node-type", nodeType);
+ } else if (topPropIndex == 1) {
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
+ + kName.get(1) + " = " + kVal.get(1) + ") ";
+ verts = g.V().has(kName.get(0), kVal.get(0)).has(kName.get(1), kVal.get(1)).has("aai-node-type", nodeType);
+ } else if (topPropIndex == 2) {
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
+ + kName.get(1) + " = " + kVal.get(1) + ", "
+ + kName.get(2) + " = " + kVal.get(2) + ") ";
+ verts = g.V().has(kName.get(0), kVal.get(0)).has(kName.get(1), kVal.get(1)).has(kName.get(2), kVal.get(2)).has("aai-node-type", nodeType);
+ } else if (topPropIndex == 3) {
+ propsAndValuesForMsg = " (" + kName.get(0) + " = " + kVal.get(0) + ", "
+ + kName.get(1) + " = " + kVal.get(1) + ", "
+ + kName.get(2) + " = " + kVal.get(2) + ", "
+ + kName.get(3) + " = " + kVal.get(3) + ") ";
+ verts = g.V().has(kName.get(0), kVal.get(0)).has(kName.get(1), kVal.get(1)).has(kName.get(2), kVal.get(2)).has(kName.get(3), kVal.get(3)).has("aai-node-type", nodeType);
+ } else {
+ throw new AAIException("AAI_6114", " We only support 4 keys per nodeType for now \n");
+ }
+ } catch (Exception ex) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(" ERROR trying to get node for: [" + propsAndValuesForMsg + "] " + LogFormatTools.getStackTop(ex));
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+ }
+
+ if (verts != null) {
+ while (verts.hasNext()) {
+ tiV = verts.next();
+ retVertList.add(tiV);
+ }
+ }
+
+ if (retVertList.size() == 0) {
+ logger.debug("DEBUG No node found for nodeType = [" + nodeType +
+ "], propsAndVal = " + propsAndValuesForMsg);
+ }
+
+ return retVertList;
+
+ }// End of getNodeJustUsingKeyParams()
+
+
+ /**
+ * Gets the node(s) just using key params.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param graph the graph
+ * @param nodeType the node type
+ * @param windowStartTime the window start time
+ * @param propsHash the props hash
+ * @param apiVersion the api version
+ * @return the nodes
+ * @throws AAIException the AAI exception
+ */
+ public ArrayList<Vertex> figureOutNodes2Check(String transId, String fromAppId,
+ Graph graph, String nodeType, long windowStartTime,
+ String propsString, EELFLogger logger) throws AAIException {
+
+ ArrayList<Vertex> retVertList = new ArrayList<Vertex>();
+ String msg = "";
+ GraphTraversal<Vertex, Vertex> tgQ = graph.traversal().V().has("aai-node-type", nodeType);
+ String qStringForMsg = "graph.traversal().V().has(\"aai-node-type\"," + nodeType + ")";
+
+ if (propsString != null && !propsString.trim().equals("")) {
+ propsString = propsString.trim();
+ int firstPipeLoc = propsString.indexOf("|");
+ if (firstPipeLoc <= 0) {
+ msg = "Bad props4Collect passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'";
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(msg);
+ exit(0);
+ }
+
+ // Note - if they're only passing on parameter, there won't be any commas
+ String[] paramArr = propsString.split(",");
+ for (int i = 0; i < paramArr.length; i++) {
+ int pipeLoc = paramArr[i].indexOf("|");
+ if (pipeLoc <= 0) {
+ msg = "Bad propsString passed: [" + propsString + "]. \n Expecting a format like, 'propName1|propVal1,propName2|propVal2'";
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(msg);
+ exit(0);
+ } else {
+ String propName = paramArr[i].substring(0, pipeLoc);
+ String propVal = paramArr[i].substring(pipeLoc + 1);
+ tgQ = tgQ.has(propName, propVal);
+ qStringForMsg = qStringForMsg + ".has(" + propName + "," + propVal + ")";
+ }
+ }
+ }
+
+ if (tgQ == null) {
+ msg = "Bad JanusGraphQuery object. ";
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logger.error(msg);
+ exit(0);
+ } else {
+ Iterator<Vertex> vertItor = tgQ;
+ while (vertItor.hasNext()) {
+ Vertex tiV = vertItor.next();
+ if (windowStartTime <= 0) {
+ // We're not applying a time-window
+ retVertList.add(tiV);
+ } else {
+ Object objTimeStamp = tiV.property("aai-created-ts").orElse(null);
+ if (objTimeStamp == null) {
+ // No timestamp - so just take it
+ retVertList.add(tiV);
+ } else {
+ long thisNodeCreateTime = (long) objTimeStamp;
+ if (thisNodeCreateTime > windowStartTime) {
+ // It is in our window, so we can take it
+ retVertList.add(tiV);
+ }
+ }
+ }
+ }
+ }
+
+ if (retVertList.size() == 0) {
+ logger.debug("DEBUG No node found for: [" + qStringForMsg + ", with aai-created-ts > " + windowStartTime);
+ }
+
+ return retVertList;
+
+ }// End of figureOutNodes2Check()
+
+
+ /**
+ * Gets the preferred dupe.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param dupeVertexList the dupe vertex list
+ * @param ver the ver
+ * @param EELFLogger the logger
+ * @return Vertex
+ * @throws AAIException the AAI exception
+ */
+ public Vertex getPreferredDupe(String transId,
+ String fromAppId, Graph g,
+ ArrayList<Vertex> dupeVertexList, String ver,
+ Boolean specialTenantRule, Loader loader, EELFLogger logger)
+ throws AAIException {
+
+ // This method assumes that it is being passed a List of vertex objects
+ // which violate our uniqueness constraints.
+
+ Vertex nullVtx = null;
+
+ if (dupeVertexList == null) {
+ return nullVtx;
+ }
+ int listSize = dupeVertexList.size();
+ if (listSize == 0) {
+ return nullVtx;
+ }
+ if (listSize == 1) {
+ return (dupeVertexList.get(0));
+ }
+
+ Vertex vtxPreferred = null;
+ Vertex currentFaveVtx = dupeVertexList.get(0);
+ for (int i = 1; i < listSize; i++) {
+ Vertex vtxB = dupeVertexList.get(i);
+ vtxPreferred = pickOneOfTwoDupes(transId, fromAppId, g,
+ currentFaveVtx, vtxB, ver, specialTenantRule, loader, logger);
+ if (vtxPreferred == null) {
+ // We couldn't choose one
+ return nullVtx;
+ } else {
+ currentFaveVtx = vtxPreferred;
+ }
+ }
+
+ return (currentFaveVtx);
+
+ } // end of getPreferredDupe()
+
+
+ /**
+ * Pick one of two dupes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param vtxA the vtx A
+ * @param vtxB the vtx B
+ * @param ver the ver
+ * @param boolean specialTenantRuleFlag flag
+ * @param EELFLogger the logger
+ * @return Vertex
+ * @throws AAIException the AAI exception
+ */
+ public Vertex pickOneOfTwoDupes(String transId,
+ String fromAppId, Graph g, Vertex vtxA,
+ Vertex vtxB, String ver, Boolean specialTenantRule, Loader loader, EELFLogger logger) throws AAIException {
+
+ Vertex nullVtx = null;
+ Vertex preferredVtx = null;
+
+ Long vidA = new Long(vtxA.id().toString());
+ Long vidB = new Long(vtxB.id().toString());
+
+ String vtxANodeType = "";
+ String vtxBNodeType = "";
+ Object obj = vtxA.<Object>property("aai-node-type").orElse(null);
+ if (obj != null) {
+ vtxANodeType = obj.toString();
+ }
+ obj = vtxB.<Object>property("aai-node-type").orElse(null);
+ if (obj != null) {
+ vtxBNodeType = obj.toString();
+ }
+
+ if (vtxANodeType.equals("") || (!vtxANodeType.equals(vtxBNodeType))) {
+ // Either they're not really dupes or there's some bad data - so
+ // don't pick one
+ return nullVtx;
+ }
+
+ // Check that node A and B both have the same key values (or else they
+ // are not dupes)
+ // (We'll check dep-node later)
+ Collection<String> keyProps = loader.introspectorFromName(vtxANodeType).getKeys();
+ Iterator<String> keyPropI = keyProps.iterator();
+ while (keyPropI.hasNext()) {
+ String propName = keyPropI.next();
+ String vtxAKeyPropVal = "";
+ obj = vtxA.<Object>property(propName).orElse(null);
+ if (obj != null) {
+ vtxAKeyPropVal = obj.toString();
+ }
+ String vtxBKeyPropVal = "";
+ obj = vtxB.<Object>property(propName).orElse(null);
+ if (obj != null) {
+ vtxBKeyPropVal = obj.toString();
+ }
+
+ if (vtxAKeyPropVal.equals("")
+ || (!vtxAKeyPropVal.equals(vtxBKeyPropVal))) {
+ // Either they're not really dupes or they are missing some key
+ // data - so don't pick one
+ return nullVtx;
+ }
+ }
+
+ // Collect the vid's and aai-node-types of the vertices that each vertex
+ // (A and B) is connected to.
+ ArrayList<String> vtxIdsConn2A = new ArrayList<String>();
+ ArrayList<String> vtxIdsConn2B = new ArrayList<String>();
+ HashMap<String, String> nodeTypesConn2A = new HashMap<String, String>();
+ HashMap<String, String> nodeTypesConn2B = new HashMap<String, String>();
+
+ ArrayList<String> retArr = new ArrayList<String>();
+ Iterator<Edge> eAI = vtxA.edges(Direction.BOTH);
+ while (eAI.hasNext()) {
+ Edge ed = eAI.next();
+ Vertex tmpVtx;
+ if (vtxA.equals(ed.inVertex())) {
+ tmpVtx = ed.outVertex();
+ } else {
+ tmpVtx = ed.inVertex();
+ }
+ if (tmpVtx == null) {
+ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ } else {
+ String conVid = tmpVtx.id().toString();
+ String nt = "";
+ obj = tmpVtx.<Object>property("aai-node-type").orElse(null);
+ if (obj != null) {
+ nt = obj.toString();
+ }
+ nodeTypesConn2A.put(nt, conVid);
+ vtxIdsConn2A.add(conVid);
+ }
+ }
+
+ Iterator<Edge> eBI = vtxB.edges(Direction.BOTH);
+ while (eBI.hasNext()) {
+ Edge ed = eBI.next();
+ Vertex tmpVtx;
+
+ if (vtxB.equals(ed.inVertex())) {
+ tmpVtx = ed.outVertex();
+ } else {
+ tmpVtx = ed.inVertex();
+ }
+ if (tmpVtx == null) {
+ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ } else {
+ String conVid = tmpVtx.id().toString();
+ String nt = "";
+ obj = tmpVtx.<Object>property("aai-node-type").orElse(null);
+ if (obj != null) {
+ nt = obj.toString();
+ }
+ nodeTypesConn2B.put(nt, conVid);
+ vtxIdsConn2B.add(conVid);
+ }
+ }
+
+ // 1 - If this kind of node needs a dependent node for uniqueness, then
+ // verify that they both nodes point to the same dependent
+ // node (otherwise they're not really duplicates)
+ // Note - there are sometimes more than one dependent node type since
+ // one nodeType can be used in different ways. But for a
+ // particular node, it will only have one dependent node that
+ // it's connected to.
+ Collection<String> depNodeTypes = loader.introspectorFromName(vtxANodeType).getDependentOn();
+ if (depNodeTypes.isEmpty()) {
+ // This kind of node is not dependent on any other. That is ok.
+ } else {
+ String depNodeVtxId4A = "";
+ String depNodeVtxId4B = "";
+ Iterator<String> iter = depNodeTypes.iterator();
+ while (iter.hasNext()) {
+ String depNodeType = iter.next();
+ if (nodeTypesConn2A.containsKey(depNodeType)) {
+ // This is the dependent node type that vertex A is using
+ depNodeVtxId4A = nodeTypesConn2A.get(depNodeType);
+ }
+ if (nodeTypesConn2B.containsKey(depNodeType)) {
+ // This is the dependent node type that vertex B is using
+ depNodeVtxId4B = nodeTypesConn2B.get(depNodeType);
+ }
+ }
+ if (depNodeVtxId4A.equals("")
+ || (!depNodeVtxId4A.equals(depNodeVtxId4B))) {
+ // Either they're not really dupes or there's some bad data - so
+ // don't pick either one
+ return nullVtx;
+ }
+ }
+
+ if (vtxIdsConn2A.size() == vtxIdsConn2B.size()) {
+ // 2 - If they both have edges to all the same vertices, then return
+ // the one with the lower vertexId.
+
+ // OR (2b)-- if this is the SPECIAL case -- of
+ // "tenant|vserver vs. tenant|service-subscription"
+ // then we pick/prefer the one that's connected to
+ // the service-subscription. AAI-8172
+ boolean allTheSame = true;
+ Iterator<String> iter = vtxIdsConn2A.iterator();
+ while (iter.hasNext()) {
+ String vtxIdConn2A = iter.next();
+ if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
+ allTheSame = false;
+ break;
+ }
+ }
+
+ if (allTheSame) {
+ if (vidA < vidB) {
+ preferredVtx = vtxA;
+ } else {
+ preferredVtx = vtxB;
+ }
+ } else if (specialTenantRule) {
+ // They asked us to apply a special rule if it applies
+ if (vtxIdsConn2A.size() == 2 && vtxANodeType.equals("tenant")) {
+ // We're dealing with two tenant nodes which each just have
+ // two connections. One must be the parent (cloud-region)
+ // which we check in step 1 above. If one connects to
+ // a vserver and the other connects to a service-subscription,
+ // our special rule is to keep the one connected
+ // to the
+ if (nodeTypesConn2A.containsKey("vserver") && nodeTypesConn2B.containsKey("service-subscription")) {
+ String infMsg = " WARNING >>> we are using the special tenant rule to choose to " +
+ " delete tenant vtxId = " + vidA + ", and keep tenant vtxId = " + vidB;
+ System.out.println(infMsg);
+ logger.info(infMsg);
+ preferredVtx = vtxB;
+ } else if (nodeTypesConn2B.containsKey("vserver") && nodeTypesConn2A.containsKey("service-subscription")) {
+ String infMsg = " WARNING >>> we are using the special tenant rule to choose to " +
+ " delete tenant vtxId = " + vidB + ", and keep tenant vtxId = " + vidA;
+ System.out.println(infMsg);
+ logger.info(infMsg);
+ preferredVtx = vtxA;
+ }
+ }
+ }
+ } else if (vtxIdsConn2A.size() > vtxIdsConn2B.size()) {
+ // 3 - VertexA is connected to more things than vtxB.
+ // We'll pick VtxA if its edges are a superset of vtxB's edges.
+ boolean missingOne = false;
+ Iterator<String> iter = vtxIdsConn2B.iterator();
+ while (iter.hasNext()) {
+ String vtxIdConn2B = iter.next();
+ if (!vtxIdsConn2A.contains(vtxIdConn2B)) {
+ missingOne = true;
+ break;
+ }
+ }
+ if (!missingOne) {
+ preferredVtx = vtxA;
+ }
+ } else if (vtxIdsConn2B.size() > vtxIdsConn2A.size()) {
+ // 4 - VertexB is connected to more things than vtxA.
+ // We'll pick VtxB if its edges are a superset of vtxA's edges.
+ boolean missingOne = false;
+ Iterator<String> iter = vtxIdsConn2A.iterator();
+ while (iter.hasNext()) {
+ String vtxIdConn2A = iter.next();
+ if (!vtxIdsConn2B.contains(vtxIdConn2A)) {
+ missingOne = true;
+ break;
+ }
+ }
+ if (!missingOne) {
+ preferredVtx = vtxB;
+ }
+ } else {
+ preferredVtx = nullVtx;
+ }
+
+ return (preferredVtx);
+
+ } // end of pickOneOfTwoDupes()
+
+
+ /**
+ * Group verts by dep nodes.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param g the g
+ * @param version the version
+ * @param nType the n type
+ * @param passedVertList the passed vert list
+ * @param dbMaps the db maps
+ * @return the hash map
+ * @throws AAIException the AAI exception
+ */
+ private HashMap<String, ArrayList<Vertex>> groupVertsByDepNodes(
+ String transId, String fromAppId, Graph g, String version,
+ String nType, ArrayList<Vertex> passedVertList, Loader loader)
+ throws AAIException {
+
+ // Given a list of JanusGraph Vertices, group them together by dependent
+ // nodes. Ie. if given a list of ip address nodes (assumed to all
+ // have the same key info) they might sit under several different
+ // parent vertices.
+ // Under Normal conditions, there would only be one per parent -- but
+ // we're trying to find duplicates - so we allow for the case
+ // where more than one is under the same parent node.
+
+ HashMap<String, ArrayList<Vertex>> retHash = new HashMap<String, ArrayList<Vertex>>();
+ GraphTraversalSource gts = g.traversal();
+ if (passedVertList != null) {
+ Iterator<Vertex> iter = passedVertList.iterator();
+ while (iter.hasNext()) {
+ Vertex thisVert = iter.next();
+ Vertex parentVtx = getConnectedParent(gts, thisVert);
+ if (parentVtx != null) {
+ String parentVid = parentVtx.id().toString();
+ if (retHash.containsKey(parentVid)) {
+ // add this vert to the list for this parent key
+ retHash.get(parentVid).add(thisVert);
+ } else {
+ // This is the first one we found on this parent
+ ArrayList<Vertex> vList = new ArrayList<Vertex>();
+ vList.add(thisVert);
+ retHash.put(parentVid, vList);
+ }
+ }
+ }
+ }
+ return retHash;
+
+ }// end of groupVertsByDepNodes()
+
+
+ private Vertex getConnectedParent(GraphTraversalSource g,
+ Vertex startVtx) throws AAIException {
+
+ Vertex parentVtx = null;
+ // This traversal does not assume a parent/child edge direction
+ Iterator<Vertex> vertI = g.V(startVtx).union(__.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).outV(), __.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).inV());
+ while (vertI != null && vertI.hasNext()) {
+ // Note - there better only be one!
+ parentVtx = vertI.next();
+ }
+ return parentVtx;
+
+ }// End of getConnectedParent()
+
+
+ /**
+ * Delete non keepers if appropriate.
+ *
+ * @param g the g
+ * @param dupeInfoList the dupe info string
+ * @param logger the EELFLogger
+ * @return the boolean
+ */
+ private Boolean deleteNonKeepers(Graph g,
+ ArrayList<String> dupeInfoList, EELFLogger logger) {
+
+ // This assumes that each dupeInfoString is in the format of
+ // pipe-delimited vid's followed by either "keepVid=xyz" or "keepVid=UNDETERMINED"
+ // ie. "3456|9880|keepVid=3456"
+
+ boolean didADelFlag = false;
+ for (int n = 0; n < dupeInfoList.size(); n++) {
+ String dupeInfoString = dupeInfoList.get(n);
+ boolean tmpFlag = deleteNonKeeperForOneSet(g, dupeInfoString, logger);
+ didADelFlag = tmpFlag | didADelFlag;
+ }
+
+ return didADelFlag;
+
+ }// end of deleteNonKeepers()
+
+
+ /**
+ * Delete non keepers if appropriate.
+ *
+ * @param g the g
+ * @param dupeSetStr the dupe string
+ * @param logger the EELFLogger
+ * @return the boolean
+ */
+ private Boolean deleteNonKeeperForOneSet(Graph g,
+ String dupeInfoString, EELFLogger logger) {
+
+ Boolean deletedSomething = false;
+ // This assumes that each dupeInfoString is in the format of
+ // pipe-delimited vid's followed by either "keepVid=xyz" or "keepVid=UNDETERMINED"
+ // ie. "3456|9880|keepVid=3456"
+
+
+ String[] dupeArr = dupeInfoString.split("\\|");
+ ArrayList<String> idArr = new ArrayList<String>();
+ int lastIndex = dupeArr.length - 1;
+ for (int i = 0; i <= lastIndex; i++) {
+ if (i < lastIndex) {
+ // This is not the last entry, it is one of the dupes,
+ String vidString = dupeArr[i];
+ idArr.add(vidString);
+ } else {
+ // This is the last entry which should tell us if we have a
+ // preferred keeper
+ String prefString = dupeArr[i];
+ if (prefString.equals("KeepVid=UNDETERMINED")) {
+ // They sent us a bad string -- nothing should be deleted if
+ // no dupe could be tagged as preferred.
+ return false;
+ } else {
+ // If we know which to keep, then the prefString should look
+ // like, "KeepVid=12345"
+ String[] prefArr = prefString.split("=");
+ if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
+ String emsg = "Bad format. Expecting KeepVid=999999";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(emsg);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+ return false;
+ } else {
+ String keepVidStr = prefArr[1];
+ if (idArr.contains(keepVidStr)) {
+ idArr.remove(keepVidStr);
+ // So now, the idArr should just contain the vid's
+ // that we want to remove.
+ for (int x = 0; x < idArr.size(); x++) {
+ boolean okFlag = true;
+ String thisVid = idArr.get(x);
+ try {
+ long longVertId = Long.parseLong(thisVid);
+ Vertex vtx = g.traversal().V(longVertId).next();
+ String msg = "--->>> We will delete node with VID = " + thisVid + " <<<---";
+ System.out.println(msg);
+ logger.info(msg);
+ vtx.remove();
+ } catch (Exception e) {
+ okFlag = false;
+ String emsg = "ERROR trying to delete VID = " + thisVid + ", [" + e + "]";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(emsg);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+ }
+ if (okFlag) {
+ String infMsg = " DELETED VID = " + thisVid;
+ logger.info(infMsg);
+ System.out.println(infMsg);
+ deletedSomething = true;
+ }
+ }
+ } else {
+ String emsg = "ERROR - Vertex Id to keep not found in list of dupes. dupeInfoString = ["
+ + dupeInfoString + "]";
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(emsg);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+ System.out.println(emsg);
+ return false;
+ }
+ }
+ }// else we know which one to keep
+ }// else last entry
+ }// for each vertex in a group
+
+ return deletedSomething;
+
+ }// end of deleteNonKeeperForOneSet()
+
+
+ /**
+ * Get values of the key properties for a node.
+ *
+ * @param tvx the vertex to pull the properties from
+ * @param keyPropertyNames ArrayList (ordered) of key prop names
+ * @param logger the EELFLogger
+ * @return a hashMap of the propertyNames/values
+ */
+ private HashMap<String, Object> getNodeKeyVals(Vertex tvx,
+ ArrayList<String> keyPropNamesArr, EELFLogger logger) {
+
+ HashMap<String, Object> retHash = new HashMap<String, Object>();
+ Iterator<String> propItr = keyPropNamesArr.iterator();
+ while (propItr.hasNext()) {
+ String propName = propItr.next();
+ if (tvx != null) {
+ Object propValObj = tvx.property(propName).orElse(null);
+ retHash.put(propName, propValObj);
+ }
+ }
+ return retHash;
+
+ }// End of getNodeKeyVals()
+
+
+ /**
+ * Get values of the key properties for a node as a single string
+ *
+ * @param tvx the vertex to pull the properties from
+ * @param keyPropertyNames collection of key prop names
+ * @param logger the EELFLogger
+ * @return a String of concatenated values
+ */
+ private String getNodeKeyValString(Vertex tvx,
+ ArrayList<String> keyPropNamesArr, EELFLogger logger) {
+
+ // -- NOTE -- for what we're using this for, we would need to
+ // guarantee that the properties are always in the same order
+
+ String retString = "";
+ Iterator<String> propItr = keyPropNamesArr.iterator();
+ while (propItr.hasNext()) {
+ String propName = propItr.next();
+ if (tvx != null) {
+ Object propValObj = tvx.property(propName).orElse(null);
+ retString = " " + retString + propValObj.toString();
+ }
+ }
+ return retString;
+
+ }// End of getNodeKeyValString()
+
+
+ /**
+ * Find duplicate sets from two dupe runs.
+ *
+ * @param firstPassDupeSets from the first pass
+ * @param secondPassDupeSets from the second pass
+ * @param EELFLogger logger
+ * @return commonDupeSets that are common to both passes and have a determined keeper
+ */
+ private ArrayList<String> figureWhichDupesStillNeedFixing(ArrayList<String> firstPassDupeSets,
+ ArrayList<String> secondPassDupeSets, EELFLogger logger) {
+
+ ArrayList<String> common2BothSet = new ArrayList<String>();
+
+ // We just want to look for entries from the first set which have identical (almost)
+ // entries in the secondary set. I say "almost" because the order of the
+ // vid's to delete may be in a different order, but we only want to use it if
+ // they have all the same values. Note also - we're just looking for
+ // the sets where we have a candidate to delete.
+
+ // The duplicate-set Strings are in this format:
+ // "1234|5678|keepVid=UNDETERMINED" (if there were 2 dupes, and we
+ // couldn't figure out which one to keep)
+ // or, "100017|200027|30037|keepVid=30037" (if there were 3 dupes and we
+ // thought the third one was the one that should survive)
+
+ if (firstPassDupeSets == null || firstPassDupeSets.isEmpty()
+ || secondPassDupeSets == null || secondPassDupeSets.isEmpty()) {
+ // If either set is empty, then our return list has to be empty too
+ return common2BothSet;
+ }
+
+ boolean needToParse = false;
+ for (int x = 0; x < secondPassDupeSets.size(); x++) {
+ String secPassDupeSetStr = secondPassDupeSets.get(x);
+ if (secPassDupeSetStr.endsWith("UNDETERMINED")) {
+ // This is a set of dupes where we could not pick one
+ // to delete - so don't include it on our list for
+ // fixing.
+ } else if (firstPassDupeSets.contains(secPassDupeSetStr)) {
+ // We have lucked out and do not even need to parse this since
+ // it was in the other array with any dupes listed in the same order
+ // This is actually the most common scenario since there is
+ // usually only one dupe, so order doesn't matter.
+ common2BothSet.add(secPassDupeSetStr);
+ } else {
+ // We'll need to do some parsing to check this one
+ needToParse = true;
+ }
+ }
+
+ if (needToParse) {
+ // Make a hash from the first and second Pass data
+ // where the key is the vid to KEEP and the value is an
+ // array of (String) vids that would get deleted.
+ HashMap<String, ArrayList<String>> firstPassHash = makeKeeperHashOfDupeStrings(firstPassDupeSets, common2BothSet, logger);
+
+ HashMap<String, ArrayList<String>> secPassHash = makeKeeperHashOfDupeStrings(secondPassDupeSets, common2BothSet, logger);
+
+ // Loop through the secondPass data and keep the ones
+ // that check out against the firstPass set.
+ for (Map.Entry<String, ArrayList<String>> entry : secPassHash.entrySet()) {
+ boolean skipThisOne = false;
+ String secKey = entry.getKey();
+ ArrayList<String> secList = entry.getValue();
+ if (!firstPassHash.containsKey(secKey)) {
+ // The second pass found this delete candidate, but not the first pass
+ skipThisOne = true;
+ } else {
+ // They both think they should keep this VID, check the associated deletes for it.
+ ArrayList<String> firstList = firstPassHash.get(secKey);
+ for (int z = 0; z < secList.size(); z++) {
+ if (!firstList.contains(secList.get(z))) {
+ // The first pass did not think this needed to be deleted
+ skipThisOne = true;
+ }
+ }
+ }
+ if (!skipThisOne) {
+ // Put the string back together and pass it back
+ // Not beautiful, but no time to make it nice right now...
+ // Put it back in the format: "3456|9880|keepVid=3456"
+ String thisDelSetStr = "";
+ for (int z = 0; z < secList.size(); z++) {
+ if (z == 0) {
+ thisDelSetStr = secList.get(z);
+ } else {
+ thisDelSetStr = thisDelSetStr + "|" + secList.get(z);
+ }
+ }
+ thisDelSetStr = thisDelSetStr + "|keepVid=" + secKey;
+ common2BothSet.add(thisDelSetStr);
+ }
+ }
+
+ }
+ return common2BothSet;
+
+ }// figureWhichDupesStillNeedFixing
+
+
+ private HashMap<String, ArrayList<String>> makeKeeperHashOfDupeStrings(ArrayList<String> dupeSets,
+ ArrayList<String> excludeSets, EELFLogger logger) {
+
+ HashMap<String, ArrayList<String>> keeperHash = new HashMap<String, ArrayList<String>>();
+
+ for (int x = 0; x < dupeSets.size(); x++) {
+ String tmpSetStr = dupeSets.get(x);
+ if (excludeSets.contains(tmpSetStr)) {
+ // This isn't one of the ones we needed to parse.
+ continue;
+ }
+
+ String[] dupeArr = tmpSetStr.split("\\|");
+ ArrayList<String> delIdArr = new ArrayList<String>();
+ int lastIndex = dupeArr.length - 1;
+ for (int i = 0; i <= lastIndex; i++) {
+ if (i < lastIndex) {
+ // This is not the last entry, it is one of the dupes
+ delIdArr.add(dupeArr[i]);
+ } else {
+ // This is the last entry which should tell us if we
+ // have a preferred keeper and how many dupes we had
+ String prefString = dupeArr[i];
+ if (i == 1) {
+ // There was only one dupe, so if we were gonna find
+ // it, we would have found it above with no parsing.
+ } else if (prefString.equals("KeepVid=UNDETERMINED")) {
+ // This one had no determined keeper, so we don't
+ // want it.
+ } else {
+ // If we know which to keep, then the prefString
+ // should look like, "KeepVid=12345"
+ String[] prefArr = prefString.split("=");
+ if (prefArr.length != 2
+ || (!prefArr[0].equals("KeepVid"))) {
+ String infMsg = "Bad format in figureWhichDupesStillNeedFixing(). Expecting " +
+ " KeepVid=999999 but string looks like: [" + tmpSetStr + "]";
+ System.out.println(infMsg);
+ logger.info(infMsg);
+ } else {
+ keeperHash.put(prefArr[0], delIdArr);
+ }
+ }
+ }
+ }
+ }
+
+ return keeperHash;
+
+ }// End makeHashOfDupeStrings()
+
+
+ /**
+ * Get values of the key properties for a node.
+ *
+ * @param g the g
+ * @param dupeInfoString
+ * @param logger the EELFLogger
+ * @return void
+ */
+ private void showNodeDetailsForADupeSet(Graph g, String dupeInfoString, EELFLogger logger) {
+
+ // dang... parsing this string once again...
+
+ String[] dupeArr = dupeInfoString.split("\\|");
+ int lastIndex = dupeArr.length - 1;
+ for (int i = 0; i <= lastIndex; i++) {
+ if (i < lastIndex) {
+ // This is not the last entry, it is one of the dupes,
+ String vidString = dupeArr[i];
+ long longVertId = Long.parseLong(vidString);
+ Vertex vtx = g.traversal().V(longVertId).next();
+ showNodeInfo(logger, vtx, false);
+ } else {
+ // This is the last entry which should tell us if we have a
+ // preferred keeper
+ String prefString = dupeArr[i];
+ if (prefString.equals("KeepVid=UNDETERMINED")) {
+ String msg = " Our algorithm cannot choose from among these, so they will all be kept. -------\n";
+ System.out.println(msg);
+ logger.info(msg);
+ } else {
+ // If we know which to keep, then the prefString should look
+ // like, "KeepVid=12345"
+ String[] prefArr = prefString.split("=");
+ if (prefArr.length != 2 || (!prefArr[0].equals("KeepVid"))) {
+ String emsg = "Bad format. Expecting KeepVid=999999";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(emsg);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+ } else {
+ String keepVidStr = prefArr[1];
+ String msg = " vid = " + keepVidStr + " is the one that we would KEEP. ------\n";
+ System.out.println(msg);
+ logger.info(msg);
+ }
+ }
+ }
+ }
+
+ }// End of showNodeDetailsForADupeSet()
+
+ private int graphIndex = 1;
+
+ public JanusGraph setupGraph(EELFLogger logger) {
+
+ JanusGraph JanusGraph = null;
+
+
+ try (InputStream inputStream = new FileInputStream(AAIConstants.REALTIME_DB_CONFIG);) {
+
+ Properties properties = new Properties();
+ properties.load(inputStream);
+
+ if ("inmemory".equals(properties.get("storage.backend"))) {
+ JanusGraph = AAIGraph.getInstance().getGraph();
+ graphType = "inmemory";
+ } else {
+ JanusGraph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(DupeTool.class.getSimpleName()).withGraphType("realtime" + graphIndex).buildConfiguration());
+ graphIndex++;
+ }
+ } catch (Exception e) {
+ logger.error("Unable to open the graph", e);
+ }
+
+ return JanusGraph;
+ }
+
+ public void closeGraph(JanusGraph graph, EELFLogger logger) {
+
+ try {
+ if ("inmemory".equals(graphType)) {
+ return;
+ }
+ if (graph != null && graph.isOpen()) {
+ graph.tx().close();
+ graph.close();
+ }
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
+ logger.warn("WARNING from final graph.shutdown()", ex);
+ }
+ }
+}
+
diff --git a/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java
new file mode 100644
index 0000000..790bfa1
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/ForceDeleteTool.java
@@ -0,0 +1,875 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.Scanner;
+import java.util.UUID;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.onap.aai.dbmap.AAIGraphConfig;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.edges.enums.EdgeProperty;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraph;
+
+
+
+public class ForceDeleteTool {
+ private static final String FROMAPPID = "AAI-DB";
+ private static final String TRANSID = UUID.randomUUID().toString();
+
+ private static String graphType = "realdb";
+
+ public static boolean SHOULD_EXIT_VM = true;
+
+ public static int EXIT_VM_STATUS_CODE = -1;
+
+ public static void exit(int statusCode){
+ if(SHOULD_EXIT_VM){
+ System.exit(1);
+ }
+ EXIT_VM_STATUS_CODE = statusCode;
+ }
+
+ /*
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args) {
+
+ //SWGK 01/21/2016 - To suppress the warning message when the tool is run from the Terminal.
+
+ System.setProperty("aai.service.name", ForceDelete.class.getSimpleName());
+ // Set the logging file properties to be used by EELFManager
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_FORCE_DELETE_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ EELFLogger logger = EELFManager.getInstance().getLogger(ForceDeleteTool.class.getSimpleName());
+ MDC.put("logFilenameAppender", ForceDeleteTool.class.getSimpleName());
+
+ LoggingContext.init();
+ LoggingContext.partnerName(FROMAPPID);
+ LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.component("forceDeleteTool");
+ LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.targetServiceName("main");
+ LoggingContext.requestId(TRANSID);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ String actionVal = "";
+ String userIdVal = "";
+ String dataString = "";
+ Boolean displayAllVidsFlag = false; // Note - This should rarely be needed
+ Boolean overRideProtection = false; // This should rarely be used - it overrides all our new checking
+ long vertexIdLong = 0;
+ String edgeIdStr = "";
+ String argStr4Msg = "";
+
+ if (args != null && args.length > 0) {
+ // They passed some arguments in that will affect processing
+ for (int i = 0; i < args.length; i++) {
+ String thisArg = args[i];
+ argStr4Msg = argStr4Msg + " " + thisArg;
+
+ if (thisArg.equals("-action")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -action option. ");
+ exit(0);
+ }
+ actionVal = args[i];
+ argStr4Msg = argStr4Msg + " " + actionVal;
+ }
+ else if (thisArg.equals("-userId")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -userId option. ");
+ exit(0);
+ }
+ userIdVal = args[i];
+ argStr4Msg = argStr4Msg + " " + userIdVal;
+ }
+ else if (thisArg.equals("-overRideProtection")) {
+ overRideProtection = true;
+ }
+ else if (thisArg.equals("-DISPLAY_ALL_VIDS")) {
+ displayAllVidsFlag = true;
+ }
+ else if (thisArg.equals("-vertexId")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -vertexId option. ");
+ exit(0);
+ }
+ String nextArg = args[i];
+ argStr4Msg = argStr4Msg + " " + nextArg;
+ try {
+ vertexIdLong = Long.parseLong(nextArg);
+ } catch (Exception e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error("Bad value passed with -vertexId option: ["
+ + nextArg + "]");
+ exit(0);
+ }
+ }
+ else if (thisArg.equals("-params4Collect")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -params4Collect option. ");
+ exit(0);
+ }
+ dataString = args[i];
+ argStr4Msg = argStr4Msg + " " + dataString;
+ }
+ else if (thisArg.equals("-edgeId")) {
+ i++;
+ if (i >= args.length) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" No value passed with -edgeId option. ");
+ exit(0);
+ }
+ String nextArg = args[i];
+ argStr4Msg = argStr4Msg + " " + nextArg;
+ edgeIdStr = nextArg;
+ }
+ else {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(" Unrecognized argument passed to ForceDeleteTool: ["
+ + thisArg + "]. ");
+ logger.error(" Valid values are: -action -userId -vertexId -edgeId -overRideProtection -params4Collect -DISPLAY_ALL_VIDS");
+ exit(0);
+ }
+ }
+ }
+
+ if( !actionVal.equals("COLLECT_DATA") && !actionVal.equals("DELETE_NODE") && !actionVal.equals("DELETE_EDGE")){
+ String emsg = "Bad action parameter [" + actionVal + "] passed to ForceDeleteTool(). Valid values = COLLECT_DATA or DELETE_NODE or DELETE_EDGE\n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+
+ if( actionVal.equals("DELETE_NODE") && vertexIdLong == 0 ){
+ String emsg = "ERROR: No vertex ID passed on DELETE_NODE request. \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+ else if( actionVal.equals("DELETE_EDGE") && edgeIdStr.equals("")){
+ String emsg = "ERROR: No edge ID passed on DELETE_EDGE request. \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+
+
+ userIdVal = userIdVal.trim();
+ if( (userIdVal.length() < 6) || userIdVal.toUpperCase().equals("AAIADMIN") ){
+ String emsg = "Bad userId parameter [" + userIdVal + "] passed to ForceDeleteTool(). must be not empty and not aaiadmin \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+
+ String msg = "";
+ JanusGraph graph = null;
+ try {
+ AAIConfig.init();
+ System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n");
+ graph = setupGraph(logger);
+ if( graph == null ){
+ String emsg = "could not get graph object in ForceDeleteTool() \n";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logger.error(emsg);
+ exit(0);
+ }
+ }
+ catch (AAIException e1) {
+ msg = e1.getErrorObject().toString();
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logger.error(msg);
+ exit(0);
+ }
+ catch (Exception e2) {
+ msg = e2.toString();
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logger.error(msg);
+ exit(0);
+ }
+
+ msg = "ForceDelete called by: userId [" + userIdVal + "] with these params: [" + argStr4Msg + "]";
+ System.out.println(msg);
+ logger.info(msg);
+
+ ForceDelete fd = new ForceDelete(graph);
+ if( actionVal.equals("COLLECT_DATA") ){
+ // When doing COLLECT_DATA, we expect them to either pass the vertexId or
+ // that the dataString string to be comma separated name value pairs like this:
+ // "propName1|propVal1,propName2|propVal2" etc. We will look for a node or nodes
+ // that have properties that ALL match what was passed in.
+ GraphTraversal<Vertex, Vertex> g = null;
+ String qStringForMsg = "";
+ int resCount = 0;
+ if( vertexIdLong > 0 ){
+ // They know which vertex they want to look at
+ qStringForMsg = "graph.vertices(" + vertexIdLong + ")";
+ Iterator <Vertex> vtxItr = graph.vertices( vertexIdLong );
+ if( vtxItr != null && vtxItr.hasNext() ) {
+ Vertex vtx = vtxItr.next();
+ fd.showNodeInfo( logger, vtx, displayAllVidsFlag );
+ resCount++;
+ }
+ }
+ else {
+ // we need to find the node or nodes based on the dataString
+ int firstPipeLoc = dataString.indexOf("|");
+ if( firstPipeLoc <= 0 ){
+ msg = "Must use the -params4Collect option when collecting data with data string in a format like: 'propName1|propVal1,propName2|propVal2'";
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(msg);
+ exit(0);
+ }
+ g = graph.traversal().V();
+ qStringForMsg = " graph.traversal().V()";
+ // Note - if they're only passing one parameter, there won't be any commas
+ String [] paramArr = dataString.split(",");
+ for( int i = 0; i < paramArr.length; i++ ){
+ int pipeLoc = paramArr[i].indexOf("|");
+ if( pipeLoc <= 0 ){
+ msg = "Must use the -params4Collect option when collecting data with data string in a format like: 'propName1|propVal1,propName2|propVal2'";
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(msg);
+ exit(0);
+ }
+ else {
+ String propName = paramArr[i].substring(0,pipeLoc);
+ String propVal = paramArr[i].substring(pipeLoc + 1);
+ g = g.has(propName,propVal);
+ qStringForMsg = qStringForMsg + ".has(" + propName + "," + propVal + ")";
+ }
+ }
+
+ if( (g != null)){
+ Iterator<Vertex> vertItor = g;
+ while( vertItor.hasNext() ){
+ resCount++;
+ Vertex v = vertItor.next();
+ fd.showNodeInfo( logger, v, displayAllVidsFlag );
+ int descendantCount = fd.countDescendants( logger, v, 0 );
+ String infMsg = " Found " + descendantCount + " descendant nodes \n";
+ System.out.println( infMsg );
+ logger.info( infMsg );
+ }
+ }
+ else {
+ msg = "Bad JanusGraphQuery object. ";
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.error(msg);
+ exit(0);
+ }
+ }
+
+ String infMsg = "\n\n Found: " + resCount + " nodes for this query: [" + qStringForMsg + "]\n";
+ System.out.println( infMsg );
+ logger.info( infMsg );
+ }
+ else if( actionVal.equals("DELETE_NODE") ){
+ Iterator <Vertex> vtxItr = graph.vertices( vertexIdLong );
+ if( vtxItr != null && vtxItr.hasNext() ) {
+ Vertex vtx = vtxItr.next();
+ fd.showNodeInfo( logger, vtx, displayAllVidsFlag );
+ int descendantCount = fd.countDescendants( logger, vtx, 0 );
+ String infMsg = " Found " + descendantCount + " descendant nodes. Note - forceDelete does not cascade to " +
+ " child nodes, but they may become unreachable after the delete. \n";
+ System.out.println( infMsg );
+ logger.info( infMsg );
+
+ int edgeCount = fd.countEdges( logger, vtx );
+
+ infMsg = " Found total of " + edgeCount + " edges incident on this node. \n";
+ System.out.println( infMsg );
+ logger.info( infMsg );
+
+ if( fd.getNodeDelConfirmation(logger, userIdVal, vtx, descendantCount, edgeCount, overRideProtection) ){
+ vtx.remove();
+ graph.tx().commit();
+ infMsg = ">>>>>>>>>> Removed node with vertexId = " + vertexIdLong;
+ logger.info( infMsg );
+ System.out.println(infMsg);
+ }
+ else {
+ infMsg = " Delete Cancelled. ";
+ System.out.println(infMsg);
+ logger.info( infMsg );
+ }
+ }
+ else {
+ String infMsg = ">>>>>>>>>> Vertex with vertexId = " + vertexIdLong + " not found.";
+ System.out.println( infMsg );
+ logger.info( infMsg );
+ }
+ }
+ else if( actionVal.equals("DELETE_EDGE") ){
+ Edge thisEdge = null;
+ Iterator <Edge> edItr = graph.edges( edgeIdStr );
+ if( edItr != null && edItr.hasNext() ) {
+ thisEdge = edItr.next();
+ }
+
+ if( thisEdge == null ){
+ String infMsg = ">>>>>>>>>> Edge with edgeId = " + edgeIdStr + " not found.";
+ logger.info( infMsg );
+ System.out.println(infMsg);
+ exit(0);
+ }
+
+ if( fd.getEdgeDelConfirmation(logger, userIdVal, thisEdge, overRideProtection) ){
+ thisEdge.remove();
+ graph.tx().commit();
+ String infMsg = ">>>>>>>>>> Removed edge with edgeId = " + edgeIdStr;
+ logger.info( infMsg );
+ System.out.println(infMsg);
+ }
+ else {
+ String infMsg = " Delete Cancelled. ";
+ System.out.println(infMsg);
+ logger.info( infMsg );
+ }
+ exit(0);
+ }
+ else {
+ String emsg = "Unknown action parameter [" + actionVal + "] passed to ForceDeleteTool(). Valid values = COLLECT_DATA, DELETE_NODE or DELETE_EDGE \n";
+ System.out.println(emsg);
+ logger.info( emsg );
+ exit(0);
+ }
+
+ closeGraph(graph, logger);
+ exit(0);
+
+ }// end of main()
+
+ public static class ForceDelete {
+
+ private final int MAXDESCENDENTDEPTH = 15;
+ private final JanusGraph graph;
+ public ForceDelete(JanusGraph graph) {
+ this.graph = graph;
+ }
+ public void showNodeInfo(EELFLogger logger, Vertex tVert, Boolean displayAllVidsFlag ){
+
+ try {
+ Iterator<VertexProperty<Object>> pI = tVert.properties();
+ String infStr = ">>> Found Vertex with VertexId = " + tVert.id() + ", properties: ";
+ System.out.println( infStr );
+ logger.info(infStr);
+ while( pI.hasNext() ){
+ VertexProperty<Object> tp = pI.next();
+ infStr = " [" + tp.key() + "|" + tp.value() + "] ";
+ System.out.println( infStr );
+ logger.info(infStr);
+ }
+
+ ArrayList <String> retArr = collectEdgeInfoForNode( logger, tVert, displayAllVidsFlag );
+ for( String infoStr : retArr ){
+ System.out.println( infoStr );
+ logger.info(infoStr);
+ }
+ }
+ catch (Exception e){
+ String warnMsg = " -- Error -- trying to display edge info. [" + e.getMessage() + "]";
+ System.out.println( warnMsg );
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.warn(warnMsg);
+ LoggingContext.successStatusFields();
+ }
+
+ }// End of showNodeInfo()
+
+
+ public void showPropertiesForEdge( EELFLogger logger, Edge tEd ){
+ String infMsg = "";
+ if( tEd == null ){
+ infMsg = "null Edge object passed to showPropertiesForEdge()";
+ System.out.print(infMsg);
+ logger.info(infMsg);
+ return;
+ }
+
+ // Try to show the edge properties
+ try {
+ infMsg =" Label for this Edge = [" + tEd.label() + "] ";
+ System.out.print(infMsg);
+ logger.info(infMsg);
+
+ infMsg =" EDGE Properties for edgeId = " + tEd.id() + ": ";
+ System.out.print(infMsg);
+ logger.info(infMsg);
+ Iterator <String> pI = tEd.keys().iterator();
+ while( pI.hasNext() ){
+ String propKey = pI.next();
+ infMsg = "Prop: [" + propKey + "], val = ["
+ + tEd.property(propKey) + "] ";
+ System.out.print(infMsg);
+ logger.info(infMsg);
+ }
+ }
+ catch( Exception ex ){
+ infMsg = " Could not retrieve properties for this edge. exMsg = ["
+ + ex.getMessage() + "] ";
+ System.out.println( infMsg );
+ logger.info(infMsg);
+ }
+
+ // Try to show what's connected to the IN side of this Edge
+ try {
+ infMsg = " Looking for the Vertex on the IN side of the edge: ";
+ System.out.print(infMsg);
+ logger.info(infMsg);
+ Vertex inVtx = tEd.inVertex();
+ Iterator<VertexProperty<Object>> pI = inVtx.properties();
+ String infStr = ">>> Found Vertex with VertexId = " + inVtx.id()
+ + ", properties: ";
+ System.out.println( infStr );
+ logger.info(infStr);
+ while( pI.hasNext() ){
+ VertexProperty<Object> tp = pI.next();
+ infStr = " [" + tp.key() + "|" + tp.value() + "] ";
+ System.out.println( infStr );
+ logger.info(infStr);
+ }
+ }
+ catch( Exception ex ){
+ infMsg = " Could not retrieve vertex data for the IN side of "
+ + "the edge. exMsg = [" + ex.getMessage() + "] ";
+ System.out.println( infMsg );
+ logger.info(infMsg);
+ }
+
+ // Try to show what's connected to the OUT side of this Edge
+ try {
+ infMsg = " Looking for the Vertex on the OUT side of the edge: ";
+ System.out.print(infMsg);
+ logger.info(infMsg);
+ Vertex outVtx = tEd.outVertex();
+ Iterator<VertexProperty<Object>> pI = outVtx.properties();
+ String infStr = ">>> Found Vertex with VertexId = " + outVtx.id()
+ + ", properties: ";
+ System.out.println( infStr );
+ logger.info(infStr);
+ while( pI.hasNext() ){
+ VertexProperty<Object> tp = pI.next();
+ infStr = " [" + tp.key() + "|" + tp.value() + "] ";
+ System.out.println( infStr );
+ logger.info(infStr);
+ }
+ }
+ catch( Exception ex ){
+ infMsg = " Could not retrieve vertex data for the OUT side of "
+ + "the edge. exMsg = [" + ex.getMessage() + "] ";
+ System.out.println( infMsg );
+ logger.info(infMsg);
+ }
+
+ }// end showPropertiesForEdge()
+
+
+
+ public ArrayList <String> collectEdgeInfoForNode( EELFLogger logger, Vertex tVert, boolean displayAllVidsFlag ){
+ ArrayList <String> retArr = new ArrayList <String> ();
+ Direction dir = Direction.OUT;
+ for ( int i = 0; i <= 1; i++ ){
+ if( i == 1 ){
+ // Second time through we'll look at the IN edges.
+ dir = Direction.IN;
+ }
+ Iterator <Edge> eI = tVert.edges(dir);
+ if( ! eI.hasNext() ){
+ retArr.add("No " + dir + " edges were found for this vertex. ");
+ }
+ while( eI.hasNext() ){
+ Edge ed = eI.next();
+ String edId = ed.id().toString();
+ String lab = ed.label();
+ Vertex vtx = null;
+ if( dir == Direction.OUT ){
+ // get the vtx on the "other" side
+ vtx = ed.inVertex();
+ }
+ else {
+ // get the vtx on the "other" side
+ vtx = ed.outVertex();
+ }
+ if( vtx == null ){
+ retArr.add(" >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ }
+ else {
+ String nType = vtx.<String>property("aai-node-type").orElse(null);
+ if( displayAllVidsFlag ){
+ // This should rarely be needed
+ String vid = vtx.id().toString();
+ retArr.add("Found an " + dir + " edge (" + lab + ") with EDGE-ID = " + edId +
+ ", between this vertex and a [" + nType + "] node with VtxId = " + vid );
+ }
+ else {
+ // This is the normal case
+ retArr.add("Found an " + dir + " edge (" + lab + ") between this vertex and a [" + nType + "] node. ");
+ }
+ }
+ }
+ }
+ return retArr;
+
+ }// end of collectEdgeInfoForNode()
+
+
+ public int countEdges( EELFLogger logger, Vertex vtx ){
+ int edgeCount = 0;
+ try {
+ Iterator<Edge> edgesItr = vtx.edges(Direction.BOTH);
+ while( edgesItr.hasNext() ){
+ edgesItr.next();
+ edgeCount++;
+ }
+ }
+ catch (Exception e) {
+ String wMsg = "-- ERROR -- Stopping the counting of edges because of Exception [" + e.getMessage() + "]";
+ System.out.println( wMsg );
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.warn( wMsg );
+ LoggingContext.successStatusFields();
+ }
+ return edgeCount;
+
+ }// end of countEdges()
+
+
+ public int countDescendants(EELFLogger logger, Vertex vtx, int levelVal ){
+ int totalCount = 0;
+ int thisLevel = levelVal + 1;
+
+ if( thisLevel > MAXDESCENDENTDEPTH ){
+ String wMsg = "Warning -- Stopping the counting of descendents because we reached the max depth of " + MAXDESCENDENTDEPTH;
+ System.out.println( wMsg );
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.warn( wMsg );
+ return totalCount;
+ }
+
+ try {
+ Iterator <Vertex> vertI = graph.traversal().V(vtx).union(__.outE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.OUT.toString()).inV(), __.inE().has(EdgeProperty.CONTAINS.toString(), AAIDirection.IN.toString()).outV());
+ while( vertI != null && vertI.hasNext() ){
+ totalCount++;
+ Vertex childVtx = vertI.next();
+ totalCount = totalCount + countDescendants( logger, childVtx, thisLevel );
+ }
+ }
+ catch (Exception e) {
+ String wMsg = "Error -- Stopping the counting of descendents because of Exception [" + e.getMessage() + "]";
+ System.out.println( wMsg );
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.warn( wMsg );
+ LoggingContext.successStatusFields();
+
+ }
+
+ return totalCount;
+ }// end of countDescendants()
+
+
+ public boolean getEdgeDelConfirmation( EELFLogger logger, String uid, Edge ed,
+ Boolean overRideProtection ) {
+
+ showPropertiesForEdge( logger, ed );
+ System.out.print("\n Are you sure you want to delete this EDGE? (y/n): ");
+ Scanner s = new Scanner(System.in);
+ s.useDelimiter("");
+ String confirm = s.next();
+ s.close();
+
+ if (!confirm.equalsIgnoreCase("y")) {
+ String infMsg = " User [" + uid + "] has chosen to abandon this delete request. ";
+ System.out.println("\n" + infMsg);
+ logger.info(infMsg);
+ return false;
+ }
+ else {
+ String infMsg = " User [" + uid + "] has confirmed this delete request. ";
+ System.out.println("\n" + infMsg);
+ logger.info(infMsg);
+ return true;
+ }
+
+ } // End of getEdgeDelConfirmation()
+
+
+ public boolean getNodeDelConfirmation( EELFLogger logger, String uid, Vertex vtx, int edgeCount,
+ int descendantCount, Boolean overRideProtection ) {
+ String thisNodeType = "";
+ try {
+ thisNodeType = vtx.<String>property("aai-node-type").orElse(null);
+ }
+ catch ( Exception nfe ){
+ // Let the user know something is going on - but they can confirm the delete if they want to.
+ String infMsg = " -- WARNING -- could not get an aai-node-type for this vertex. -- WARNING -- ";
+ System.out.println( infMsg );
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.warn( infMsg );
+ LoggingContext.successStatusFields();
+ }
+
+ String ntListString = "";
+ String maxDescString = "";
+ String maxEdgeString = "";
+
+ int maxDescCount = 10; // default value
+ int maxEdgeCount = 10; // default value
+ ArrayList <String> protectedNTypes = new ArrayList <String> ();
+ protectedNTypes.add("cloud-region"); // default value
+
+ try {
+ ntListString = AAIConfig.get("aai.forceDel.protected.nt.list");
+ maxDescString = AAIConfig.get("aai.forceDel.protected.descendant.count");
+ maxEdgeString = AAIConfig.get("aai.forceDel.protected.edge.count");
+ }
+ catch ( Exception nfe ){
+ // Don't worry, we will use default values
+ String infMsg = "-- WARNING -- could not get aai.forceDel.protected values from aaiconfig.properties -- will use default values. ";
+ System.out.println( infMsg );
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logger.warn( infMsg );
+ LoggingContext.successStatusFields();
+ }
+
+ if( maxDescString != null && !maxDescString.equals("") ){
+ try {
+ maxDescCount = Integer.parseInt(maxDescString);
+ }
+ catch ( Exception nfe ){
+ // Don't worry, we will leave "maxDescCount" set to the default value
+ }
+ }
+
+ if( maxEdgeString != null && !maxEdgeString.equals("") ){
+ try {
+ maxEdgeCount = Integer.parseInt(maxEdgeString);
+ }
+ catch ( Exception nfe ){
+ // Don't worry, we will leave "maxEdgeCount" set to the default value
+ }
+ }
+
+ if( ntListString != null && !ntListString.trim().equals("") ){
+ String [] nodeTypes = ntListString.split("\\|");
+ for( int i = 0; i < nodeTypes.length; i++ ){
+ protectedNTypes.add(nodeTypes[i]);
+ }
+ }
+
+ boolean giveProtOverRideMsg = false;
+ boolean giveProtErrorMsg = false;
+ if( descendantCount > maxDescCount ){
+ // They are trying to delete a node with a lots of descendants
+ String infMsg = " >> WARNING >> This node has more descendant edges than the max ProtectedDescendantCount: " + edgeCount + ". Max = " +
+ maxEdgeCount + ". It can be DANGEROUS to delete one of these. << WARNING << ";
+ System.out.println(infMsg);
+ logger.info(infMsg);
+ if( ! overRideProtection ){
+ // They cannot delete this kind of node without using the override option
+ giveProtErrorMsg = true;
+ }
+ else {
+ giveProtOverRideMsg = true;
+ }
+ }
+
+ if( edgeCount > maxEdgeCount ){
+ // They are trying to delete a node with a lot of edges
+ String infMsg = " >> WARNING >> This node has more edges than the max ProtectedEdgeCount: " + edgeCount + ". Max = " +
+ maxEdgeCount + ". It can be DANGEROUS to delete one of these. << WARNING << ";
+ System.out.println(infMsg);
+ logger.info(infMsg);
+ if( ! overRideProtection ){
+ // They cannot delete this kind of node without using the override option
+ giveProtErrorMsg = true;
+ }
+ else {
+ giveProtOverRideMsg = true;
+ }
+ }
+
+ if( thisNodeType != null && !thisNodeType.equals("") && protectedNTypes.contains(thisNodeType) ){
+ // They are trying to delete a protected Node Type
+ String infMsg = " >> WARNING >> This node is a PROTECTED NODE-TYPE (" + thisNodeType + "). " +
+ " It can be DANGEROUS to delete one of these. << WARNING << ";
+ System.out.println(infMsg);
+ logger.info(infMsg);
+ if( ! overRideProtection ){
+ // They cannot delete this kind of node without using the override option
+ giveProtErrorMsg = true;
+ }
+ else {
+ giveProtOverRideMsg = true;
+ }
+ }
+
+ if( giveProtOverRideMsg ){
+ String infMsg = " !!>> WARNING >>!! you are using the overRideProtection parameter which will let you do this potentially dangerous delete.";
+ System.out.println("\n" + infMsg);
+ logger.info(infMsg);
+ }
+ else if( giveProtErrorMsg ) {
+ String errMsg = " ERROR >> this kind of node can only be deleted if you pass the overRideProtection parameter.";
+ System.out.println("\n" + errMsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logger.error(errMsg);
+ LoggingContext.successStatusFields();
+ return false;
+ }
+
+ System.out.print("\n Are you sure you want to do this delete? (y/n): ");
+ Scanner s = new Scanner(System.in);
+ s.useDelimiter("");
+ String confirm = s.next();
+ s.close();
+
+ if (!confirm.equalsIgnoreCase("y")) {
+ String infMsg = " User [" + uid + "] has chosen to abandon this delete request. ";
+ System.out.println("\n" + infMsg);
+ logger.info(infMsg);
+ return false;
+ }
+ else {
+ String infMsg = " User [" + uid + "] has confirmed this delete request. ";
+ System.out.println("\n" + infMsg);
+ logger.info(infMsg);
+ return true;
+ }
+
+ } // End of getNodeDelConfirmation()
+ }
+
+ public static JanusGraph setupGraph(EELFLogger logger){
+
+ JanusGraph janusGraph = null;
+
+ try (InputStream inputStream = new FileInputStream(AAIConstants.REALTIME_DB_CONFIG);){
+
+ Properties properties = new Properties();
+ properties.load(inputStream);
+
+ if("inmemory".equals(properties.get("storage.backend"))){
+ janusGraph = AAIGraph.getInstance().getGraph();
+ graphType = "inmemory";
+ } else {
+ janusGraph = JanusGraphFactory.open(
+ new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG)
+ .forService(ForceDeleteTool.class.getSimpleName())
+ .withGraphType("realtime1")
+ .buildConfiguration()
+ );
+ }
+ } catch (Exception e) {
+ logger.error("Unable to open the graph", LogFormatTools.getStackTop(e));
+ }
+
+ return janusGraph;
+ }
+
+ public static void closeGraph(JanusGraph graph, EELFLogger logger){
+
+ try {
+ if("inmemory".equals(graphType)) {
+ return;
+ }
+ if( graph != null && graph.isOpen() ){
+ graph.tx().close();
+ graph.close();
+ }
+ } catch (Exception ex) {
+ // Don't throw anything because JanusGraph sometimes is just saying that the graph is already closed{
+ logger.warn("WARNING from final graph.shutdown()", ex);
+ }
+ }
+}
+
diff --git a/src/main/java/org/onap/aai/dbgen/GraphMLTokens.java b/src/main/java/org/onap/aai/dbgen/GraphMLTokens.java
new file mode 100644
index 0000000..d43b57f
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/GraphMLTokens.java
@@ -0,0 +1,56 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen;
+
+/**
+ * A collection of tokens used for GraphML related data.
+ */
+public class GraphMLTokens {
+ public static final String GRAPHML = "graphml";
+ public static final String XMLNS = "xmlns";
+ public static final String GRAPHML_XMLNS = "http://graphml.graphdrawing.org/xmlns";
+ public static final String G = "G";
+ public static final String EDGEDEFAULT = "edgedefault";
+ public static final String DIRECTED = "directed";
+ public static final String KEY = "key";
+ public static final String FOR = "for";
+ public static final String ID = "id";
+ public static final String ATTR_NAME = "attr.name";
+ public static final String ATTR_TYPE = "attr.type";
+ public static final String GRAPH = "graph";
+ public static final String NODE = "node";
+ public static final String EDGE = "edge";
+ public static final String SOURCE = "source";
+ public static final String TARGET = "target";
+ public static final String DATA = "data";
+ public static final String LABEL = "label";
+ public static final String STRING = "string";
+ public static final String FLOAT = "float";
+ public static final String DOUBLE = "double";
+ public static final String LONG = "long";
+ public static final String BOOLEAN = "boolean";
+ public static final String INT = "int";
+ public static final String ARRAY = "array";
+ public static final String SET = "set";
+ public static final String LIST = "list";
+ public static final String ITEM = "item";
+ public static final String _DEFAULT = "_default";
+
+}
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
new file mode 100644
index 0000000..c0f8ee9
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaMod.java
@@ -0,0 +1,177 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen.schemamod;
+
+import java.util.Properties;
+
+import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.UniquePropertyCheck;
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+public class SchemaMod {
+
+ private final LoaderFactory loaderFactory;
+
+ private final SchemaVersions schemaVersions;
+
+ public SchemaMod(LoaderFactory loaderFactory, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.schemaVersions = schemaVersions;
+ }
+
+ public void execute(String[] args) {
+
+ // Set the logging file properties to be used by EELFManager
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_SCHEMA_MOD_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+
+ EELFLogger logger = EELFManager.getInstance().getLogger(UniquePropertyCheck.class.getSimpleName());
+ MDC.put("logFilenameAppender", SchemaMod.class.getSimpleName());
+
+ // NOTE -- We're just working with properties that are used for NODES
+ // for now.
+ String propName = "";
+ String targetDataType = "";
+ String targetIndexInfo = "";
+ String preserveDataFlag = "";
+
+ String usageString = "Usage: SchemaMod propertyName targetDataType targetIndexInfo preserveDataFlag \n";
+ if (args.length != 4) {
+ String emsg = "Four Parameters are required. \n" + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ } else {
+ propName = args[0];
+ targetDataType = args[1];
+ targetIndexInfo = args[2];
+ preserveDataFlag = args[3];
+ }
+
+ if (propName.equals("")) {
+ String emsg = "Bad parameter - propertyName cannot be empty. \n" + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ } else if (!targetDataType.equals("String") && !targetDataType.equals("Set<String>")
+ && !targetDataType.equals("Integer") && !targetDataType.equals("Long")
+ && !targetDataType.equals("Boolean")) {
+ String emsg = "Unsupported targetDataType. We only support String, Set<String>, Integer, Long or Boolean for now.\n"
+ + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ } else if (!targetIndexInfo.equals("uniqueIndex") && !targetIndexInfo.equals("index")
+ && !targetIndexInfo.equals("noIndex")) {
+ String emsg = "Unsupported IndexInfo. We only support: 'uniqueIndex', 'index' or 'noIndex'.\n"
+ + usageString;
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+
+ try {
+ AAIConfig.init();
+ ErrorLogHelper.loadProperties();
+ } catch (Exception ae) {
+ String emsg = "Problem with either AAIConfig.init() or ErrorLogHelper.LoadProperties(). ";
+ logAndPrint(logger, emsg + "[" + ae.getMessage() + "]");
+ System.exit(1);
+ }
+
+ // Give a big warning if the DbMaps.PropertyDataTypeMap value does not
+ // agree with what we're doing
+ String warningMsg = "";
+
+ if (!warningMsg.equals("")) {
+ logAndPrint(logger, "\n>>> WARNING <<<< ");
+ logAndPrint(logger, ">>> " + warningMsg + " <<<");
+ }
+
+ logAndPrint(logger, ">>> Processing will begin in 5 seconds (unless interrupted). <<<");
+ try {
+ // Give them a chance to back out of this
+ Thread.sleep(5000);
+ } catch (java.lang.InterruptedException ie) {
+ logAndPrint(logger, " DB Schema Update has been aborted. ");
+ System.exit(1);
+ }
+
+ logAndPrint(logger, " ---- NOTE --- about to open graph (takes a little while)\n");
+
+ SchemaVersion version = schemaVersions.getDefaultVersion();
+ QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ ModelType introspectorFactoryType = ModelType.MOXY;
+ Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ TransactionalGraphEngine engine = null;
+ try {
+ engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+ SchemaModInternal internal = new SchemaModInternal(engine, logger, propName, targetDataType, targetIndexInfo, new Boolean(preserveDataFlag));
+ internal.execute();
+ engine.startTransaction();
+ engine.tx().close();
+ logAndPrint(logger, "------ Completed the SchemaMod -------- ");
+ } catch (Exception e) {
+ String emsg = "Not able to complete the requested SchemaMod \n";
+ logAndPrint(logger, e.getMessage());
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+ }
+ /**
+ * Log and print.
+ *
+ * @param logger the logger
+ * @param msg the msg
+ */
+ protected void logAndPrint(EELFLogger logger, String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+ public static void main(String[] args) {
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+ SchemaMod schemaMod = new SchemaMod(loaderFactory, schemaVersions);
+ schemaMod.execute(args);
+
+ System.exit(0);
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java
new file mode 100644
index 0000000..b5ce16b
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/schemamod/SchemaModInternal.java
@@ -0,0 +1,317 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen.schemamod;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.util.FormatDate;
+import org.onap.aai.util.UniquePropertyCheck;
+
+import com.att.eelf.configuration.EELFLogger;
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+
+public class SchemaModInternal {
+ private static final String FROMAPPID = "AAI-UTILS";
+ private final String TRANSID = UUID.randomUUID().toString();
+ private final TransactionalGraphEngine engine;
+ private final String propName;
+ private final Class<?> type;
+ private final String indexType;
+ private final boolean preserveData;
+ private final Cardinality cardinality;
+ private final EELFLogger logger;
+
+ public SchemaModInternal(TransactionalGraphEngine engine, EELFLogger logger, String propName, String type, String indexType, boolean preserveData) {
+ this.engine = engine;
+ this.propName = propName;
+ this.type = determineClass(type);
+ this.indexType = indexType;
+ this.preserveData = preserveData;
+ this.cardinality = determineCardinality(type);
+ this.logger = logger;
+ }
+
+
+ private Class<?> determineClass(String type) {
+ final Class<?> result;
+ if (type.equals("String")) {
+ result = String.class;
+ } else if (type.equals("Set<String>")) {
+ result = String.class;
+ } else if (type.equals("Integer")) {
+ result = Integer.class;
+ } else if (type.equals("Boolean")) {
+ result = Boolean.class;
+ } else if (type.equals("Character")) {
+ result = Character.class;
+ } else if (type.equals("Long")) {
+ result = Long.class;
+ } else if (type.equals("Float")) {
+ result = Float.class;
+ } else if (type.equals("Double")) {
+ result = Double.class;
+ } else {
+ String emsg = "Not able translate the targetDataType [" + type + "] to a Class variable.\n";
+ logAndPrint(logger, emsg);
+ throw new RuntimeException(emsg);
+ }
+
+ return result;
+ }
+ private Cardinality determineCardinality(String type) {
+ if (type.equals("Set<String>")) {
+ return Cardinality.SET;
+ } else {
+ return Cardinality.SINGLE;
+ }
+ }
+ public void execute() {
+ JanusGraphManagement graphMgt = null;
+ boolean success = false;
+ try {
+ // Make sure this property is in the DB.
+ graphMgt = engine.asAdmin().getManagementSystem();
+ if (graphMgt == null) {
+ String emsg = "Not able to get a graph Management object in SchemaMod.java\n";
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+ PropertyKey origPropKey = graphMgt.getPropertyKey(propName);
+ if (origPropKey == null) {
+ String emsg = "The propName = [" + propName + "] is not defined in our graph. ";
+ logAndPrint(logger, emsg);
+ System.exit(1);
+ }
+
+ if (indexType.equals("uniqueIndex")) {
+ // Make sure the data in the property being changed can have a
+ // unique-index put on it.
+ // Ie. if there are duplicate values, we will not be able to
+ // migrate the data back into the property.
+
+
+ Graph grTmp = engine.tx();
+ if( grTmp == null ){
+ grTmp = engine.startTransaction();
+ }
+ // This is good to know in the logs
+ logAndPrint(logger, "-- Starting UniquePropertyCheck. (this may take a loooong time) --");
+
+ Boolean foundDupesFlag = UniquePropertyCheck.runTheCheckForUniqueness(TRANSID, FROMAPPID,
+ grTmp, propName, logger);
+ if (foundDupesFlag) {
+ logAndPrint(logger,
+ "\n\n!!!!!! >> Cannot add a uniqueIndex for the property: [" + propName
+ + "] because duplicate values were found. See the log for details on which"
+ + " nodes have this value. \nThey will need to be resolved (by updating those values to new"
+ + " values or deleting unneeded nodes) using the standard REST-API \n");
+ System.exit(1);
+ }
+ logAndPrint(logger, "-- Finished UniquePropertyCheck. "); // This is good to know in the logs
+ }
+
+
+ // ---- If we made it to here - we must be OK with making this change
+
+ // Rename this property to a backup name (old name with "retired_"
+ // appended plus a dateStr)
+ FormatDate fd = new FormatDate("MMddHHmm", "GMT");
+ String dteStr= fd.getDateTime();
+
+ String retiredName = propName + "-" + dteStr + "-RETIRED";
+ graphMgt.changeName(origPropKey, retiredName);
+
+ // Create a new property using the original property name and the
+ // targetDataType
+ PropertyKey freshPropKey = graphMgt.makePropertyKey(propName).dataType(type)
+ .cardinality(cardinality).make();
+
+ // Create the appropriate index (if any)
+ if (indexType.equals("uniqueIndex")) {
+ String freshIndexName = propName + dteStr;
+ graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).unique().buildCompositeIndex();
+ } else if (indexType.equals("index")) {
+ String freshIndexName = propName + dteStr;
+ graphMgt.buildIndex(freshIndexName, Vertex.class).addKey(freshPropKey).buildCompositeIndex();
+ }
+
+ logAndPrint(logger, "Committing schema changes with graphMgt.commit()");
+ graphMgt.commit();
+ engine.commit();
+ Graph grTmp2 = engine.startTransaction();
+
+
+ // For each node that has this property, update the new from the old
+ // and then remove the
+ // old property from that node
+ Iterator<Vertex> verts = grTmp2.traversal().V().has(retiredName);
+ int vtxCount = 0;
+ ArrayList<String> alreadySeenVals = new ArrayList<String>();
+ while (verts.hasNext()) {
+ vtxCount++;
+ Vertex tmpVtx = verts.next();
+ String tmpVid = tmpVtx.id().toString();
+ Object origVal = tmpVtx.<Object> property(retiredName).orElse(null);
+ if (preserveData) {
+ tmpVtx.property(propName, origVal);
+ if (indexType.equals("uniqueIndex")) {
+ // We're working on a property that is being used as a
+ // unique index
+ String origValStr = "";
+ if (origVal != null) {
+ origValStr = origVal.toString();
+ }
+ if (alreadySeenVals.contains(origValStr)) {
+ // This property is supposed to be unique, but we've
+ // already seen this value in this loop
+ // This should have been caught up in the first part
+ // of SchemaMod, but since it wasn't, we
+ // will just log the problem.
+ logAndPrint(logger,
+ "\n\n ---------- ERROR - could not migrate the old data [" + origValStr
+ + "] for propertyName [" + propName
+ + "] because this property is having a unique index put on it.");
+ showPropertiesAndEdges(TRANSID, FROMAPPID, tmpVtx, logger);
+ logAndPrint(logger, "-----------------------------------\n");
+ } else {
+ // Ok to add this prop in as a unique value
+ tmpVtx.property(propName, origVal);
+ logAndPrint(logger,
+ "INFO -- just did the add of the freshPropertyKey and updated it with the orig value ("
+ + origValStr + ")");
+ }
+ alreadySeenVals.add(origValStr);
+ } else {
+ // We are not working with a unique index
+ tmpVtx.property(propName, origVal);
+ logAndPrint(logger,
+ "INFO -- just did the add of the freshPropertyKey and updated it with the orig value ("
+ + origVal.toString() + ")");
+ }
+ } else {
+ // existing nodes just won't have that property anymore
+ // Not sure if we'd ever actually want to do this -- maybe
+ // we'd do this if the new
+ // data type was not compatible with the old?
+ }
+ tmpVtx.property(retiredName).remove();
+ logAndPrint(logger, "INFO -- just did the remove of the " + retiredName + " from this vertex. (vid="
+ + tmpVid + ")");
+ }
+
+ success = true;
+ } catch (Exception ex) {
+ logAndPrint(logger, "Threw a regular Exception: ");
+ logAndPrint(logger, ex.getMessage());
+ } finally {
+ if (graphMgt != null && graphMgt.isOpen()) {
+ // Any changes that worked correctly should have already done
+ // their commits.
+ graphMgt.rollback();
+ }
+ if (engine != null) {
+ if (success) {
+ engine.commit();
+ } else {
+ engine.rollback();
+ }
+ }
+ }
+ }
+
+ /**
+ * Show properties and edges.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param tVert the t vert
+ * @param logger the logger
+ */
+ private static void showPropertiesAndEdges(String transId, String fromAppId, Vertex tVert, EELFLogger logger) {
+
+ if (tVert == null) {
+ logAndPrint(logger, "Null node passed to showPropertiesAndEdges.");
+ } else {
+ String nodeType = "";
+ Object ob = tVert.<String> property("aai-node-type");
+ if (ob == null) {
+ nodeType = "null";
+ } else {
+ nodeType = ob.toString();
+ }
+
+ logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
+ logAndPrint(logger, " Property Detail: ");
+ Iterator<VertexProperty<Object>> pI = tVert.properties();
+ while (pI.hasNext()) {
+ VertexProperty<Object> tp = pI.next();
+ Object val = tp.value();
+ logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] ");
+ }
+
+ Iterator<Edge> eI = tVert.edges(Direction.BOTH);
+ if (!eI.hasNext()) {
+ logAndPrint(logger, "No edges were found for this vertex. ");
+ }
+ while (eI.hasNext()) {
+ Edge ed = eI.next();
+ String lab = ed.label();
+ Vertex vtx;
+ if (tVert.equals(ed.inVertex())) {
+ vtx = ed.outVertex();
+ } else {
+ vtx = ed.inVertex();
+ }
+ if (vtx == null) {
+ logAndPrint(logger,
+ " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ } else {
+ String nType = vtx.<String> property("aai-node-type").orElse(null);
+ String vid = vtx.id().toString();
+ logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType
+ + "] node with VtxId = " + vid);
+ }
+ }
+ }
+ } // End of showPropertiesAndEdges()
+
+ /**
+ * Log and print.
+ *
+ * @param logger the logger
+ * @param msg the msg
+ */
+ protected static void logAndPrint(EELFLogger logger, String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/dbgen/tags/Command.java b/src/main/java/org/onap/aai/dbgen/tags/Command.java
new file mode 100644
index 0000000..ac553f9
--- /dev/null
+++ b/src/main/java/org/onap/aai/dbgen/tags/Command.java
@@ -0,0 +1,25 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.dbgen.tags;
+
+@FunctionalInterface
+interface Command {
+ public abstract void execute ( ) throws Exception;
+}
diff --git a/src/main/java/org/onap/aai/interceptors/AAIContainerFilter.java b/src/main/java/org/onap/aai/interceptors/AAIContainerFilter.java
new file mode 100644
index 0000000..6fb7356
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/AAIContainerFilter.java
@@ -0,0 +1,41 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors;
+
+import org.onap.aai.util.FormatDate;
+
+import java.util.UUID;
+
+public abstract class AAIContainerFilter {
+
+ protected String genDate() {
+ FormatDate fd = new FormatDate("YYMMdd-HH:mm:ss:SSS");
+ return fd.getDateTime();
+ }
+
+ protected boolean isValidUUID(String transId) {
+ try {
+ UUID.fromString(transId);
+ } catch (IllegalArgumentException e) {
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/src/main/java/org/onap/aai/interceptors/AAIHeaderProperties.java b/src/main/java/org/onap/aai/interceptors/AAIHeaderProperties.java
new file mode 100644
index 0000000..6801aee
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/AAIHeaderProperties.java
@@ -0,0 +1,39 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors;
+
+public final class AAIHeaderProperties {
+
+ private AAIHeaderProperties(){}
+
+ public static final String REQUEST_CONTEXT = "aai-request-context";
+
+ public static final String HTTP_METHOD_OVERRIDE = "X-HTTP-Method-Override";
+
+ public static final String TRANSACTION_ID = "X-TransactionId";
+
+ public static final String FROM_APP_ID = "X-FromAppId";
+
+ public static final String AAI_TX_ID = "X-AAI-TXID";
+
+ public static final String AAI_REQUEST = "X-REQUEST";
+
+ public static final String AAI_REQUEST_TS = "X-REQUEST-TS";
+}
diff --git a/src/main/java/org/onap/aai/interceptors/package-info.java b/src/main/java/org/onap/aai/interceptors/package-info.java
new file mode 100644
index 0000000..ee9c334
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/package-info.java
@@ -0,0 +1,36 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+/**
+ * <b>Interceptors</b> package is subdivided to pre and post interceptors
+ * If you want to add an additional interceptor you would need to add
+ * the priority level to AAIRequestFilterPriority or AAIResponsePriority
+ * to give a value which indicates the order in which the interceptor
+ * will be triggered and also you will add that value like here
+ *
+ * <pre>
+ * <code>
+ * @Priority(AAIRequestFilterPriority.YOUR_PRIORITY)
+ * public class YourInterceptor extends AAIContainerFilter implements ContainerRequestFilter {
+ *
+ * }
+ * </code>
+ * </pre>
+ */
+package org.onap.aai.interceptors; \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/interceptors/post/AAIResponseFilterPriority.java b/src/main/java/org/onap/aai/interceptors/post/AAIResponseFilterPriority.java
new file mode 100644
index 0000000..146f847
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/post/AAIResponseFilterPriority.java
@@ -0,0 +1,40 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.post;
+
+/**
+ * Response Filter order is done reverse sorted
+ * so in the following case the first response filter would be
+ * HEADER_MANIPULATION, RESPONSE_TRANS_LOGGING, RESET_LOGGING_CONTEXT,
+ * and INVALID_RESPONSE_STATUS
+ */
+public final class AAIResponseFilterPriority {
+
+ private AAIResponseFilterPriority() {}
+
+ public static final int INVALID_RESPONSE_STATUS = 1000;
+
+ public static final int RESET_LOGGING_CONTEXT = 2000;
+
+ public static final int RESPONSE_TRANS_LOGGING = 3000;
+
+ public static final int HEADER_MANIPULATION = 4000;
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/post/InvalidResponseStatus.java b/src/main/java/org/onap/aai/interceptors/post/InvalidResponseStatus.java
new file mode 100644
index 0000000..7fd0b9c
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/post/InvalidResponseStatus.java
@@ -0,0 +1,65 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.post;
+
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.logging.ErrorLogHelper;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+@Priority(AAIResponseFilterPriority.INVALID_RESPONSE_STATUS)
+public class InvalidResponseStatus extends AAIContainerFilter implements ContainerResponseFilter {
+
+ @Override
+ public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext)
+ throws IOException {
+
+ if(responseContext.getStatus() == 405){
+
+ responseContext.setStatus(400);
+ AAIException e = new AAIException("AAI_3012");
+ ArrayList<String> templateVars = new ArrayList<>();
+
+ List<MediaType> mediaTypeList = new ArrayList<>();
+
+ String contentType = responseContext.getHeaderString("Content-Type");
+
+ if (contentType == null) {
+ mediaTypeList.add(MediaType.APPLICATION_XML_TYPE);
+ } else {
+ mediaTypeList.add(MediaType.valueOf(contentType));
+ }
+
+ String message = ErrorLogHelper.getRESTAPIErrorResponse(mediaTypeList, e, templateVars);
+
+ responseContext.setEntity(message);
+ }
+
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java b/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java
new file mode 100644
index 0000000..baf28ad
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/post/ResetLoggingContext.java
@@ -0,0 +1,98 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.post;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.Priority;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import javax.ws.rs.core.Response.Status;
+import javax.ws.rs.core.Response.StatusType;
+import java.io.IOException;
+
+@Priority(AAIResponseFilterPriority.RESET_LOGGING_CONTEXT)
+public class ResetLoggingContext extends AAIContainerFilter implements ContainerResponseFilter {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(ResetLoggingContext.class);
+
+ @Autowired
+ private HttpServletRequest httpServletRequest;
+
+ @Override
+ public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext)
+ throws IOException {
+
+ this.cleanLoggingContext(responseContext);
+
+ }
+
+ private void cleanLoggingContext(ContainerResponseContext responseContext) {
+ //String url = httpServletRequest.getRequestURL().toString();
+ boolean success = true;
+ String uri = httpServletRequest.getRequestURI();
+ String queryString = httpServletRequest.getQueryString();
+
+ if(queryString != null && !queryString.isEmpty()){
+ uri = uri + "?" + queryString;
+ }
+ // For now, we use the the HTTP status code,
+ // This may change, once the requirements for response codes are defined
+
+ int httpStatusCode = responseContext.getStatus();
+ if ( httpStatusCode < 100 || httpStatusCode > 599 ) {
+ httpStatusCode = Status.INTERNAL_SERVER_ERROR.getStatusCode();
+ }
+ LoggingContext.responseCode(Integer.toString(httpStatusCode));
+
+ StatusType sType = responseContext.getStatusInfo();
+ if ( sType != null ) {
+ Status.Family sFamily = sType.getFamily();
+ if ( ! ( Status.Family.SUCCESSFUL.equals(sFamily) ||
+ ( Status.NOT_FOUND.equals(Status.fromStatusCode(httpStatusCode)) ) ) ) {
+ success = false;
+ }
+ }
+ else {
+ if ( (httpStatusCode < 200 || httpStatusCode > 299) && ( ! ( Status.NOT_FOUND.equals(Status.fromStatusCode(httpStatusCode) ) ) ) ) {
+ success = false;
+ }
+ }
+ if (success) {
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LOGGER.info(uri + " call succeeded");
+ }
+ else {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LOGGER.error(uri + " call failed with responseCode=" + httpStatusCode);
+ }
+ LoggingContext.clear();
+
+
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/post/ResponseHeaderManipulation.java b/src/main/java/org/onap/aai/interceptors/post/ResponseHeaderManipulation.java
new file mode 100644
index 0000000..9d4efe7
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/post/ResponseHeaderManipulation.java
@@ -0,0 +1,64 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.post;
+
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+
+@Priority(AAIResponseFilterPriority.HEADER_MANIPULATION)
+public class ResponseHeaderManipulation extends AAIContainerFilter implements ContainerResponseFilter {
+
+ private static final String DEFAULT_XML_TYPE = MediaType.APPLICATION_XML;
+
+ @Override
+ public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext)
+ throws IOException {
+
+ updateResponseHeaders(requestContext, responseContext);
+
+ }
+
+ private void updateResponseHeaders(ContainerRequestContext requestContext,
+ ContainerResponseContext responseContext) {
+
+ responseContext.getHeaders().add(AAIHeaderProperties.AAI_TX_ID, requestContext.getProperty(AAIHeaderProperties.AAI_TX_ID));
+
+ String responseContentType = responseContext.getHeaderString("Content-Type");
+
+ if(responseContentType == null){
+ String acceptType = requestContext.getHeaderString("Accept");
+
+ if(acceptType == null || "*/*".equals(acceptType)){
+ responseContext.getHeaders().putSingle("Content-Type", DEFAULT_XML_TYPE);
+ } else {
+ responseContext.getHeaders().putSingle("Content-Type", acceptType);
+ }
+ }
+
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java b/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java
new file mode 100644
index 0000000..547a7c8
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/post/ResponseTransactionLogging.java
@@ -0,0 +1,123 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.post;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.google.gson.JsonObject;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.util.AAIConfig;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.Priority;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerResponseContext;
+import javax.ws.rs.container.ContainerResponseFilter;
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Optional;
+
+@Priority(AAIResponseFilterPriority.RESPONSE_TRANS_LOGGING)
+public class ResponseTransactionLogging extends AAIContainerFilter implements ContainerResponseFilter {
+
+ private static final EELFLogger TRANSACTION_LOGGER = EELFManager.getInstance().getLogger(ResponseTransactionLogging.class);
+
+ @Autowired
+ private HttpServletResponse httpServletResponse;
+
+ @Override
+ public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext)
+ throws IOException {
+
+ this.transLogging(requestContext, responseContext);
+
+ }
+
+ private void transLogging(ContainerRequestContext requestContext, ContainerResponseContext responseContext) {
+
+ String logValue;
+ String getValue;
+ String postValue;
+
+ try {
+ logValue = AAIConfig.get("aai.transaction.logging");
+ getValue = AAIConfig.get("aai.transaction.logging.get");
+ postValue = AAIConfig.get("aai.transaction.logging.post");
+ } catch (AAIException e) {
+ return;
+ }
+
+ String transId = requestContext.getHeaderString(AAIHeaderProperties.TRANSACTION_ID);
+ String fromAppId = requestContext.getHeaderString(AAIHeaderProperties.FROM_APP_ID);
+ String fullUri = requestContext.getUriInfo().getRequestUri().toString();
+ String requestTs = (String)requestContext.getProperty(AAIHeaderProperties.AAI_REQUEST_TS);
+
+ String httpMethod = requestContext.getMethod();
+
+ String status = Integer.toString(responseContext.getStatus());
+
+ String request = (String)requestContext.getProperty(AAIHeaderProperties.AAI_REQUEST);
+ String response = this.getResponseString(responseContext);
+
+ if (!Boolean.parseBoolean(logValue)) {
+ } else if (!Boolean.parseBoolean(getValue) && "GET".equals(httpMethod)) {
+ } else if (!Boolean.parseBoolean(postValue) && "POST".equals(httpMethod)) {
+ } else {
+
+ JsonObject logEntry = new JsonObject();
+ logEntry.addProperty("transactionId", transId);
+ logEntry.addProperty("status", status);
+ logEntry.addProperty("rqstDate", requestTs);
+ logEntry.addProperty("respDate", this.genDate());
+ logEntry.addProperty("sourceId", fromAppId + ":" + transId);
+ logEntry.addProperty("resourceId", fullUri);
+ logEntry.addProperty("resourceType", httpMethod);
+ logEntry.addProperty("rqstBuf", Objects.toString(request, ""));
+ logEntry.addProperty("respBuf", Objects.toString(response, ""));
+
+ try {
+ TRANSACTION_LOGGER.debug(logEntry.toString());
+ } catch (Exception e) {
+ ErrorLogHelper.logError("AAI_4000", "Exception writing transaction log.");
+ }
+ }
+
+ }
+
+ private String getResponseString(ContainerResponseContext responseContext) {
+ JsonObject response = new JsonObject();
+ response.addProperty("ID", responseContext.getHeaderString(AAIHeaderProperties.AAI_TX_ID));
+ response.addProperty("Content-Type", this.httpServletResponse.getContentType());
+ response.addProperty("Response-Code", responseContext.getStatus());
+ response.addProperty("Headers", responseContext.getHeaders().toString());
+ Optional<Object> entityOptional = Optional.ofNullable(responseContext.getEntity());
+ if(entityOptional.isPresent()){
+ response.addProperty("Entity", entityOptional.get().toString());
+ } else {
+ response.addProperty("Entity", "");
+ }
+ return response.toString();
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/AAIRequestFilterPriority.java b/src/main/java/org/onap/aai/interceptors/pre/AAIRequestFilterPriority.java
new file mode 100644
index 0000000..c3d9d3b
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/AAIRequestFilterPriority.java
@@ -0,0 +1,46 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+public final class AAIRequestFilterPriority {
+
+ private AAIRequestFilterPriority() {}
+
+ public static final int REQUEST_TRANS_LOGGING = 1000;
+
+ public static final int HEADER_VALIDATION = 2000;
+
+ public static final int SET_LOGGING_CONTEXT = 3000;
+
+ public static final int HTTP_HEADER = 4000;
+
+ public static final int LATEST = 4250;
+
+ public static final int AUTHORIZATION = 4500;
+
+ public static final int RETIRED_SERVICE = 5000;
+
+ public static final int VERSION = 5500;
+
+ public static final int HEADER_MANIPULATION = 6000;
+
+ public static final int REQUEST_MODIFICATION = 7000;
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java b/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java
new file mode 100644
index 0000000..afacf66
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/HeaderValidation.java
@@ -0,0 +1,91 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+import org.onap.aai.logging.ErrorLogHelper;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.HEADER_VALIDATION)
+public class HeaderValidation extends AAIContainerFilter implements ContainerRequestFilter {
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws IOException {
+
+ Optional<Response> oResp;
+
+ MultivaluedMap<String, String> headersMap = requestContext.getHeaders();
+
+ String transId = headersMap.getFirst(AAIHeaderProperties.TRANSACTION_ID);
+ String fromAppId = headersMap.getFirst(AAIHeaderProperties.FROM_APP_ID);
+
+ List<MediaType> acceptHeaderValues = requestContext.getAcceptableMediaTypes();
+
+ oResp = this.validateHeaderValuePresence(fromAppId, "AAI_4009", acceptHeaderValues);
+ if (oResp.isPresent()) {
+ requestContext.abortWith(oResp.get());
+ return;
+ }
+ oResp = this.validateHeaderValuePresence(transId, "AAI_4010", acceptHeaderValues);
+ if (oResp.isPresent()) {
+ requestContext.abortWith(oResp.get());
+ return;
+ }
+
+ if (!this.isValidUUID(transId)) {
+ transId = UUID.randomUUID().toString();
+ requestContext.getHeaders().get(AAIHeaderProperties.TRANSACTION_ID).clear();
+ requestContext.getHeaders().add(AAIHeaderProperties.TRANSACTION_ID, transId);
+ }
+
+ }
+
+ private Optional<Response> validateHeaderValuePresence(String value, String errorCode,
+ List<MediaType> acceptHeaderValues) {
+ Response response = null;
+ AAIException aaie;
+ if (value == null) {
+ aaie = new AAIException(errorCode);
+ return Optional.of(Response.status(aaie.getErrorObject().getHTTPResponseCode())
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(acceptHeaderValues, aaie, new ArrayList<>()))
+ .build());
+ }
+
+ return Optional.ofNullable(response);
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/HttpHeaderInterceptor.java b/src/main/java/org/onap/aai/interceptors/pre/HttpHeaderInterceptor.java
new file mode 100644
index 0000000..94d8ca1
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/HttpHeaderInterceptor.java
@@ -0,0 +1,55 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+
+import javax.annotation.Priority;
+import javax.ws.rs.HttpMethod;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+
+/**
+ * The Class HttpHeaderInterceptor
+ */
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.HTTP_HEADER)
+public class HttpHeaderInterceptor extends AAIContainerFilter implements ContainerRequestFilter {
+ public static final String patchMethod = "PATCH";
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+
+ MultivaluedMap<String, String> headersMap = containerRequestContext.getHeaders();
+ String overrideMethod = headersMap.getFirst(AAIHeaderProperties.HTTP_METHOD_OVERRIDE);
+ String httpMethod = containerRequestContext.getMethod();
+
+ if (HttpMethod.POST.equalsIgnoreCase(httpMethod) && patchMethod.equalsIgnoreCase(overrideMethod)) {
+ containerRequestContext.setMethod(patchMethod);
+ }
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/OneWaySslAuthorization.java b/src/main/java/org/onap/aai/interceptors/pre/OneWaySslAuthorization.java
new file mode 100644
index 0000000..6563e23
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/OneWaySslAuthorization.java
@@ -0,0 +1,81 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.Profiles;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.service.AuthorizationService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+@Provider
+@Profile(Profiles.ONE_WAY_SSL)
+@PreMatching
+@Priority(AAIRequestFilterPriority.AUTHORIZATION)
+public class OneWaySslAuthorization extends AAIContainerFilter implements ContainerRequestFilter {
+
+ @Autowired
+ private AuthorizationService authorizationService;
+
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException
+ {
+
+ String basicAuth = containerRequestContext.getHeaderString("Authorization");
+ List<MediaType> acceptHeaderValues = containerRequestContext.getAcceptableMediaTypes();
+
+ if(basicAuth == null || !basicAuth.startsWith("Basic ")){
+ Optional<Response> responseOptional = errorResponse("AAI_3300", acceptHeaderValues);
+ containerRequestContext.abortWith(responseOptional.get());
+ return;
+ }
+
+ basicAuth = basicAuth.replaceAll("Basic ", "");
+
+ if(!authorizationService.checkIfUserAuthorized(basicAuth)){
+ Optional<Response> responseOptional = errorResponse("AAI_3300", acceptHeaderValues);
+ containerRequestContext.abortWith(responseOptional.get());
+ return;
+ }
+
+ }
+
+ private Optional<Response> errorResponse(String errorCode, List<MediaType> acceptHeaderValues) {
+ AAIException aaie = new AAIException(errorCode);
+ return Optional.of(Response.status(aaie.getErrorObject().getHTTPResponseCode())
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(acceptHeaderValues, aaie, new ArrayList<>()))
+ .build());
+
+ }
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/RequestHeaderManipulation.java b/src/main/java/org/onap/aai/interceptors/pre/RequestHeaderManipulation.java
new file mode 100644
index 0000000..ee4807e
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/RequestHeaderManipulation.java
@@ -0,0 +1,62 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.Provider;
+import java.util.Collections;
+import java.util.regex.Matcher;
+
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.HEADER_MANIPULATION)
+public class RequestHeaderManipulation extends AAIContainerFilter implements ContainerRequestFilter {
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) {
+
+ String uri = requestContext.getUriInfo().getPath();
+ this.addRequestContext(uri, requestContext.getHeaders());
+
+ }
+
+ private void addRequestContext(String uri, MultivaluedMap<String, String> requestHeaders) {
+
+ String rc = "";
+
+ Matcher match = VersionInterceptor.EXTRACT_VERSION_PATTERN.matcher(uri);
+ if (match.find()) {
+ rc = match.group(1);
+ }
+
+ if (requestHeaders.containsKey(AAIHeaderProperties.REQUEST_CONTEXT)) {
+ requestHeaders.remove(AAIHeaderProperties.REQUEST_CONTEXT);
+ }
+ requestHeaders.put(AAIHeaderProperties.REQUEST_CONTEXT, Collections.singletonList(rc));
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/RequestModification.java b/src/main/java/org/onap/aai/interceptors/pre/RequestModification.java
new file mode 100644
index 0000000..9c17ffc
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/RequestModification.java
@@ -0,0 +1,77 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.interceptors.AAIContainerFilter;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.core.UriBuilder;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.REQUEST_MODIFICATION)
+public class RequestModification extends AAIContainerFilter implements ContainerRequestFilter {
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws IOException {
+
+ this.cleanDME2QueryParams(requestContext);
+
+ }
+
+ private void cleanDME2QueryParams(ContainerRequestContext request) {
+ UriBuilder builder = request.getUriInfo().getRequestUriBuilder();
+ MultivaluedMap<String, String> queries = request.getUriInfo().getQueryParameters();
+
+ String[] blacklist = { "version", "envContext", "routeOffer" };
+ Set<String> blacklistSet = Arrays.stream(blacklist).collect(Collectors.toSet());
+
+ boolean remove = true;
+
+ for (String param : blacklistSet) {
+ if (!queries.containsKey(param)) {
+ remove = false;
+ break;
+ }
+ }
+
+ if (remove) {
+ for (Map.Entry<String, List<String>> query : queries.entrySet()) {
+ String key = query.getKey();
+ if (blacklistSet.contains(key)) {
+ builder.replaceQueryParam(key);
+ }
+ }
+ }
+ request.setRequestUri(builder.build());
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java b/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java
new file mode 100644
index 0000000..b770296
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/RequestTransactionLogging.java
@@ -0,0 +1,136 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.google.gson.JsonObject;
+import org.glassfish.jersey.message.internal.ReaderWriter;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+import org.onap.aai.logging.LogFormatTools;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.HbaseSaltPrefixer;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.Priority;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.Provider;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.SecureRandom;
+import java.util.Random;
+import java.util.UUID;
+
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.REQUEST_TRANS_LOGGING)
+public class RequestTransactionLogging extends AAIContainerFilter implements ContainerRequestFilter {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(RequestTransactionLogging.class);
+
+ @Autowired
+ private HttpServletRequest httpServletRequest;
+
+ private static final String DEFAULT_CONTENT_TYPE = MediaType.APPLICATION_JSON;
+ private static final String DEFAULT_RESPONSE_TYPE = MediaType.APPLICATION_XML;
+
+ private static final String CONTENT_TYPE = "Content-Type";
+ private static final String ACCEPT = "Accept";
+ private static final String TEXT_PLAIN = "text/plain";
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws IOException {
+
+ String currentTimeStamp = genDate();
+ String fullId = this.getAAITxIdToHeader(currentTimeStamp);
+ this.addToRequestContext(requestContext, AAIHeaderProperties.AAI_TX_ID, fullId);
+ this.addToRequestContext(requestContext, AAIHeaderProperties.AAI_REQUEST, this.getRequest(requestContext, fullId));
+ this.addToRequestContext(requestContext, AAIHeaderProperties.AAI_REQUEST_TS, currentTimeStamp);
+ this.addDefaultContentType(requestContext);
+ }
+
+ private void addToRequestContext(ContainerRequestContext requestContext, String name, String aaiTxIdToHeader) {
+ requestContext.setProperty(name, aaiTxIdToHeader);
+ }
+
+ private void addDefaultContentType(ContainerRequestContext requestContext) {
+
+ MultivaluedMap<String, String> headersMap = requestContext.getHeaders();
+ String contentType = headersMap.getFirst(CONTENT_TYPE);
+ String acceptType = headersMap.getFirst(ACCEPT);
+
+ if(contentType == null || contentType.contains(TEXT_PLAIN)){
+ requestContext.getHeaders().putSingle(CONTENT_TYPE, DEFAULT_CONTENT_TYPE);
+ }
+
+ if(acceptType == null || acceptType.contains(TEXT_PLAIN)){
+ requestContext.getHeaders().putSingle(ACCEPT, DEFAULT_RESPONSE_TYPE);
+ }
+ }
+
+ private String getAAITxIdToHeader(String currentTimeStamp) {
+ String txId = UUID.randomUUID().toString();
+ try {
+ Random rand = new SecureRandom();
+ int number = rand.nextInt(99999);
+ txId = HbaseSaltPrefixer.getInstance().prependSalt(AAIConfig.get(AAIConstants.AAI_NODENAME) + "-"
+ + currentTimeStamp + "-" + number ); //new Random(System.currentTimeMillis()).nextInt(99999)
+ } catch (AAIException e) {
+ }
+
+ return txId;
+ }
+
+ private String getRequest(ContainerRequestContext requestContext, String fullId) {
+
+ JsonObject request = new JsonObject();
+ request.addProperty("ID", fullId);
+ request.addProperty("Http-Method", requestContext.getMethod());
+ request.addProperty(CONTENT_TYPE, httpServletRequest.getContentType());
+ request.addProperty("Headers", requestContext.getHeaders().toString());
+
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ InputStream in = requestContext.getEntityStream();
+
+ try {
+ if (in.available() > 0) {
+ ReaderWriter.writeTo(in, out);
+ byte[] requestEntity = out.toByteArray();
+ request.addProperty("Payload", new String(requestEntity, "UTF-8"));
+ requestContext.setEntityStream(new ByteArrayInputStream(requestEntity));
+ }
+ } catch (IOException ex) {
+ LOGGER.error("An exception occurred during the transaction logging: " + LogFormatTools.getStackTop(ex));
+ }
+
+ return request.toString();
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/RetiredInterceptor.java b/src/main/java/org/onap/aai/interceptors/pre/RetiredInterceptor.java
new file mode 100644
index 0000000..9a33b05
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/RetiredInterceptor.java
@@ -0,0 +1,150 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.service.RetiredService;
+import org.onap.aai.util.AAIConfig;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+// Can cache this so if the uri was already cached then it won't run the string
+// matching each time but only does it for the first time
+
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.RETIRED_SERVICE)
+public class RetiredInterceptor extends AAIContainerFilter implements ContainerRequestFilter {
+
+ private static final Pattern VERSION_PATTERN = Pattern.compile("v\\d+|latest");
+
+ private RetiredService retiredService;
+
+ private String basePath;
+
+ @Autowired
+ public RetiredInterceptor(RetiredService retiredService, @Value("${schema.uri.base.path}") String basePath){
+ this.retiredService = retiredService;
+ this.basePath = basePath;
+ if(!basePath.endsWith("/")){
+ this.basePath = basePath + "/";
+ }
+ }
+ @Override
+ public void filter(ContainerRequestContext containerRequestContext) throws IOException {
+
+ String requestURI = containerRequestContext.getUriInfo().getAbsolutePath().getPath();
+
+ String version = extractVersionFromPath(requestURI);
+
+ List<Pattern> retiredAllVersionList = retiredService.getRetiredAllVersionList();
+
+
+ if(checkIfUriRetired(containerRequestContext, retiredAllVersionList, version, requestURI, "")){
+ return;
+ }
+
+ List<Pattern> retiredVersionList = retiredService.getRetiredPatterns();
+
+ checkIfUriRetired(containerRequestContext, retiredVersionList, version, requestURI);
+ }
+
+ public boolean checkIfUriRetired(ContainerRequestContext containerRequestContext,
+ List<Pattern> retiredPatterns,
+ String version,
+ String requestURI,
+ String message){
+
+
+ for(Pattern retiredPattern : retiredPatterns){
+ if(retiredPattern.matcher(requestURI).matches()){
+ AAIException e;
+
+ if(message == null){
+ e = new AAIException("AAI_3007");
+ } else {
+ e = new AAIException("AAI_3015");
+ }
+
+ ArrayList<String> templateVars = new ArrayList<>();
+
+ if (templateVars.isEmpty()) {
+ templateVars.add("PUT");
+ if(requestURI != null){
+ requestURI = requestURI.replaceAll(basePath, "");
+ }
+ templateVars.add(requestURI);
+ if(message == null){
+ templateVars.add(version);
+ templateVars.add(AAIConfig.get("aai.default.api.version", ""));
+ }
+ }
+
+ Response response = Response
+ .status(e.getErrorObject().getHTTPResponseCode())
+ .entity(
+ ErrorLogHelper
+ .getRESTAPIErrorResponse(
+ containerRequestContext.getAcceptableMediaTypes(), e, templateVars
+ )
+ )
+ .build();
+
+ containerRequestContext.abortWith(response);
+
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ public boolean checkIfUriRetired(ContainerRequestContext containerRequestContext,
+ List<Pattern> retiredPatterns,
+ String version,
+ String requestURI){
+ return checkIfUriRetired(containerRequestContext, retiredPatterns, version, requestURI, null);
+ }
+
+ protected String extractVersionFromPath(String requestURI) {
+ Matcher versionMatcher = VERSION_PATTERN.matcher(requestURI);
+ String version = null;
+
+ if(versionMatcher.find()){
+ version = versionMatcher.group(0);
+ }
+ return version;
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java b/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java
new file mode 100644
index 0000000..6c3a7fc
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/SetLoggingContext.java
@@ -0,0 +1,75 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+import org.onap.aai.logging.LoggingContext;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+
+import javax.annotation.Priority;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.Provider;
+import java.io.IOException;
+
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.SET_LOGGING_CONTEXT)
+public class SetLoggingContext extends AAIContainerFilter implements ContainerRequestFilter {
+
+ @Autowired
+ private Environment environment;
+
+ @Autowired
+ private HttpServletRequest httpServletRequest;
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) throws IOException {
+
+ String uri = httpServletRequest.getRequestURI();
+ String queryString = httpServletRequest.getQueryString();
+
+ if(queryString != null && !queryString.isEmpty()){
+ uri = uri + "?" + queryString;
+ }
+
+ String httpMethod = requestContext.getMethod();
+
+ MultivaluedMap<String, String> headersMap = requestContext.getHeaders();
+
+ String transId = headersMap.getFirst(AAIHeaderProperties.TRANSACTION_ID);
+ String fromAppId = headersMap.getFirst(AAIHeaderProperties.FROM_APP_ID);
+
+ LoggingContext.init();
+ LoggingContext.requestId(transId);
+ LoggingContext.partnerName(fromAppId);
+ LoggingContext.targetEntity(environment.getProperty("spring.application.name"));
+ LoggingContext.component(fromAppId);
+ LoggingContext.serviceName(httpMethod + " " + uri);
+ LoggingContext.targetServiceName(httpMethod + " " + uri);
+ LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java b/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java
new file mode 100644
index 0000000..73b7877
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/TwoWaySslAuthorization.java
@@ -0,0 +1,187 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.auth.AAIAuthCore;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.interceptors.AAIHeaderProperties;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.restcore.HttpMethod;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
+
+import javax.annotation.Priority;
+import javax.security.auth.x500.X500Principal;
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.Provider;
+import java.security.cert.X509Certificate;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Provider
+@PreMatching
+@Priority(AAIRequestFilterPriority.AUTHORIZATION)
+@Profile("two-way-ssl")
+public class TwoWaySslAuthorization extends AAIContainerFilter implements ContainerRequestFilter {
+
+ @Autowired
+ private HttpServletRequest httpServletRequest;
+
+ @Autowired
+ private AAIAuthCore aaiAuthCore;
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) {
+
+ Optional<Response> oResp;
+
+ String uri = requestContext.getUriInfo().getAbsolutePath().getPath();
+ String httpMethod = getHttpMethod(requestContext);
+
+ List<MediaType> acceptHeaderValues = requestContext.getAcceptableMediaTypes();
+
+ Optional<String> authUser = getUser(this.httpServletRequest);
+
+ if (authUser.isPresent()) {
+ oResp = this.authorize(uri, httpMethod, acceptHeaderValues, authUser.get(),
+ this.getHaProxyUser(this.httpServletRequest), getCertIssuer(this.httpServletRequest));
+ if (oResp.isPresent()) {
+ requestContext.abortWith(oResp.get());
+ return;
+ }
+ } else {
+ AAIException aaie = new AAIException("AAI_9107");
+ requestContext
+ .abortWith(Response
+ .status(aaie.getErrorObject().getHTTPResponseCode()).entity(ErrorLogHelper
+ .getRESTAPIErrorResponseWithLogging(acceptHeaderValues, aaie, new ArrayList<>()))
+ .build());
+ }
+
+ }
+
+ private String getCertIssuer(HttpServletRequest hsr) {
+ String issuer = hsr.getHeader("X-AAI-SSL-Issuer");
+ if (issuer != null && !issuer.isEmpty()) {
+ // the haproxy header replaces the ', ' with '/' and reverses on the '/' need to undo that.
+ List<String> broken = Arrays.asList(issuer.split("/"));
+ broken = broken.stream().filter(s -> !s.isEmpty()).collect(Collectors.toList());
+ Collections.reverse(broken);
+ issuer = String.join(", ", broken);
+ } else {
+ if (hsr.getAttribute("javax.servlet.request.cipher_suite") != null) {
+ X509Certificate[] certChain = (X509Certificate[]) hsr.getAttribute("javax.servlet.request.X509Certificate");
+ if (certChain != null && certChain.length > 0) {
+ X509Certificate clientCert = certChain[0];
+ issuer = clientCert.getIssuerX500Principal().getName();
+ }
+ }
+ }
+ return issuer;
+ }
+
+ private String getHttpMethod(ContainerRequestContext requestContext) {
+ String httpMethod = requestContext.getMethod();
+ if ("POST".equalsIgnoreCase(httpMethod)
+ && "PATCH".equals(requestContext.getHeaderString(AAIHeaderProperties.HTTP_METHOD_OVERRIDE))) {
+ httpMethod = HttpMethod.MERGE_PATCH.toString();
+ }
+ if (httpMethod.equalsIgnoreCase(HttpMethod.MERGE_PATCH.toString()) || "patch".equalsIgnoreCase(httpMethod)) {
+ httpMethod = HttpMethod.PUT.toString();
+ }
+ return httpMethod;
+ }
+
+ private Optional<String> getUser(HttpServletRequest hsr) {
+ String authUser = null;
+ if (hsr.getAttribute("javax.servlet.request.cipher_suite") != null) {
+ X509Certificate[] certChain = (X509Certificate[]) hsr.getAttribute("javax.servlet.request.X509Certificate");
+
+ /*
+ * If the certificate is null or the certificate chain length is zero Then
+ * retrieve the authorization in the request header Authorization Check that it
+ * is not null and that it starts with Basic and then strip the basic portion to
+ * get the base64 credentials Check if this is contained in the AAIBasicAuth
+ * Singleton class If it is, retrieve the username associated with that
+ * credentials and set to authUser Otherwise, get the principal from certificate
+ * and use that authUser
+ */
+
+ if (certChain == null || certChain.length == 0) {
+
+ String authorization = hsr.getHeader("Authorization");
+
+ if (authorization != null && authorization.startsWith("Basic ")) {
+ authUser = authorization.replace("Basic ", "");
+ }
+
+ } else {
+ X509Certificate clientCert = certChain[0];
+ X500Principal subjectDN = clientCert.getSubjectX500Principal();
+ authUser = subjectDN.toString().toLowerCase();
+ }
+ }
+
+ return Optional.ofNullable(authUser);
+ }
+
+ private String getHaProxyUser(HttpServletRequest hsr) {
+ String haProxyUser;
+ if (Objects.isNull(hsr.getHeader("X-AAI-SSL-Client-CN"))
+ || Objects.isNull(hsr.getHeader("X-AAI-SSL-Client-OU"))
+ || Objects.isNull(hsr.getHeader("X-AAI-SSL-Client-O"))
+ || Objects.isNull(hsr.getHeader("X-AAI-SSL-Client-L"))
+ || Objects.isNull(hsr.getHeader("X-AAI-SSL-Client-ST"))
+ || Objects.isNull(hsr.getHeader("X-AAI-SSL-Client-C"))) {
+ haProxyUser = "";
+ } else {
+ haProxyUser = String.format("CN=%s, OU=%s, O=\"%s\", L=%s, ST=%s, C=%s",
+ Objects.toString(hsr.getHeader("X-AAI-SSL-Client-CN"), ""),
+ Objects.toString(hsr.getHeader("X-AAI-SSL-Client-OU"), ""),
+ Objects.toString(hsr.getHeader("X-AAI-SSL-Client-O"), ""),
+ Objects.toString(hsr.getHeader("X-AAI-SSL-Client-L"), ""),
+ Objects.toString(hsr.getHeader("X-AAI-SSL-Client-ST"), ""),
+ Objects.toString(hsr.getHeader("X-AAI-SSL-Client-C"), "")).toLowerCase();
+ }
+ return haProxyUser;
+ }
+
+ private Optional<Response> authorize(String uri, String httpMethod, List<MediaType> acceptHeaderValues,
+ String authUser, String haProxyUser, String issuer) {
+ Response response = null;
+ try {
+ if (!aaiAuthCore.authorize(authUser, uri, httpMethod, haProxyUser, issuer)) {
+ throw new AAIException("AAI_9101", "Request on " + httpMethod + " " + uri + " status is not OK");
+ }
+ } catch (AAIException e) {
+ response = Response.status(e.getErrorObject().getHTTPResponseCode())
+ .entity(ErrorLogHelper.getRESTAPIErrorResponseWithLogging(acceptHeaderValues, e, new ArrayList<>()))
+ .build();
+ }
+ return Optional.ofNullable(response);
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java b/src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java
new file mode 100644
index 0000000..f591120
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/VersionInterceptor.java
@@ -0,0 +1,101 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import javax.ws.rs.core.Response;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+@PreMatching
+@Priority(AAIRequestFilterPriority.VERSION)
+public class VersionInterceptor extends AAIContainerFilter implements ContainerRequestFilter {
+
+ public static final Pattern EXTRACT_VERSION_PATTERN = Pattern.compile("^(v[1-9][0-9]*).*$");
+
+ private final Set<String> allowedVersions;
+
+ private final SchemaVersions schemaVersions;
+
+ @Autowired
+ public VersionInterceptor(SchemaVersions schemaVersions){
+ this.schemaVersions = schemaVersions;
+ allowedVersions = schemaVersions.getVersions()
+ .stream()
+ .map(SchemaVersion::toString)
+ .collect(Collectors.toSet());
+
+ }
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) {
+
+ String uri = requestContext.getUriInfo().getPath();
+
+ if (uri.startsWith("search") || uri.startsWith("util/echo") || uri.startsWith("tools")) {
+ return;
+ }
+
+ Matcher matcher = EXTRACT_VERSION_PATTERN.matcher(uri);
+
+ String version = null;
+ if(matcher.matches()){
+ version = matcher.group(1);
+ } else {
+ requestContext.abortWith(createInvalidVersionResponse("AAI_3017", requestContext, version));
+ return;
+ }
+
+ if(!allowedVersions.contains(version)){
+ requestContext.abortWith(createInvalidVersionResponse("AAI_3016", requestContext, version));
+ }
+ }
+
+ private Response createInvalidVersionResponse(String errorCode, ContainerRequestContext context, String version) {
+ AAIException e = new AAIException(errorCode);
+ ArrayList<String> templateVars = new ArrayList<>();
+
+ if (templateVars.isEmpty()) {
+ templateVars.add(context.getMethod());
+ templateVars.add(context.getUriInfo().getPath());
+ templateVars.add(version);
+ }
+
+ String entity = ErrorLogHelper.getRESTAPIErrorResponse(context.getAcceptableMediaTypes(), e, templateVars);
+
+ return Response
+ .status(e.getErrorObject().getHTTPResponseCode())
+ .entity(entity)
+ .build();
+ }
+}
diff --git a/src/main/java/org/onap/aai/interceptors/pre/VersionLatestInterceptor.java b/src/main/java/org/onap/aai/interceptors/pre/VersionLatestInterceptor.java
new file mode 100644
index 0000000..61008b6
--- /dev/null
+++ b/src/main/java/org/onap/aai/interceptors/pre/VersionLatestInterceptor.java
@@ -0,0 +1,56 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.interceptors.pre;
+
+import org.onap.aai.interceptors.AAIContainerFilter;
+import org.onap.aai.setup.SchemaVersions;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestContext;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.PreMatching;
+import java.net.URI;
+
+@PreMatching
+@Priority(AAIRequestFilterPriority.LATEST)
+public class VersionLatestInterceptor extends AAIContainerFilter implements ContainerRequestFilter {
+
+ private final SchemaVersions schemaVersions;
+
+ @Autowired
+ public VersionLatestInterceptor(SchemaVersions schemaVersions){
+ this.schemaVersions = schemaVersions;
+ }
+
+ @Override
+ public void filter(ContainerRequestContext requestContext) {
+
+ String uri = requestContext.getUriInfo().getPath();
+
+ if(uri.startsWith("latest")){
+ String absolutePath = requestContext.getUriInfo().getAbsolutePath().toString();
+ String latest = absolutePath.replaceFirst("latest", schemaVersions.getDefaultVersion().toString());
+ requestContext.setRequestUri(URI.create(latest));
+ return;
+ }
+
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/EdgeMigrator.java b/src/main/java/org/onap/aai/migration/EdgeMigrator.java
new file mode 100644
index 0000000..99b4896
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/EdgeMigrator.java
@@ -0,0 +1,145 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.util.List;
+
+import com.google.common.collect.Multimap;
+import org.javatuples.Pair;
+
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.EdgeRuleQuery;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.edges.EdgeRule;
+import org.onap.aai.setup.SchemaVersions;
+
+/**
+ * A migration template for migrating all edge properties between "from" and "to" node from the DbedgeRules.json
+ *
+ */
+@MigrationPriority(0)
+@MigrationDangerRating(1)
+public abstract class EdgeMigrator extends Migrator {
+
+ private boolean success = true;
+
+ public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ public EdgeMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, List<Pair<String, String>> nodePairList) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+
+ /**
+ * Do not override this method as an inheritor of this class
+ */
+ @Override
+ public void run() {
+
+ executeModifyOperation();
+
+ }
+
+ /**
+ * This is where inheritors should add their logic
+ */
+ protected void executeModifyOperation() {
+
+ changeEdgeProperties();
+
+ }
+
+ protected void changeEdgeLabels() {
+ //TODO: when json file has edge label as well as edge property changes
+ }
+
+
+
+ protected void changeEdgeProperties() {
+ try {
+ List<Pair<String, String>> nodePairList = this.getAffectedNodePairTypes();
+ for (Pair<String, String> nodePair : nodePairList) {
+
+ String NODE_A = nodePair.getValue0();
+ String NODE_B = nodePair.getValue1();
+ Multimap<String, EdgeRule> result = edgeIngestor.getRules(new EdgeRuleQuery.Builder(NODE_A, NODE_B).build());
+
+ GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V();
+ /*
+ * Find Out-Edges from Node A to Node B and change them
+ * Also Find Out-Edges from Node B to Node A and change them
+ */
+ g.union(__.has(AAIProperties.NODE_TYPE, NODE_A).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_B)),
+ __.has(AAIProperties.NODE_TYPE, NODE_B).outE().where(__.inV().has(AAIProperties.NODE_TYPE, NODE_A)))
+ .sideEffect(t -> {
+ Edge e = t.get();
+ try {
+ Vertex out = e.outVertex();
+ Vertex in = e.inVertex();
+ if (out == null || in == null) {
+ logger.error(
+ e.id() + " invalid because one vertex was null: out=" + out + " in=" + in);
+ } else {
+ if (result.containsKey(e.label())) {
+ EdgeRule rule = result.get(e.label()).iterator().next();
+ e.properties().forEachRemaining(prop -> prop.remove());
+ edgeSerializer.addProperties(e, rule);
+ } else {
+ logger.info("found vertices connected by unkwown label: out=" + out + " label="
+ + e.label() + " in=" + in);
+ }
+ }
+ } catch (Exception e1) {
+ throw new RuntimeException(e1);
+ }
+ }).iterate();
+ }
+
+ } catch (Exception e) {
+ logger.error("error encountered", e);
+ success = false;
+ }
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ /**
+ * List of node pairs("from" and "to"), you would like EdgeMigrator to migrate from json files
+ * @return
+ */
+ public abstract List<Pair<String, String>> getAffectedNodePairTypes() ;
+
+}
diff --git a/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java
new file mode 100644
index 0000000..616ff02
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/EdgeSwingMigrator.java
@@ -0,0 +1,288 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.javatuples.Pair;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+/**
+ * A migration template for "swinging" edges that terminate on an old-node to a new target node.
+ * That is, given an oldNode and a newNode we will swing edges that terminate on the
+ * oldNode and terminate them on the newNode (actually we drop the old edges and add new ones).
+ *
+ *
+ * We allow the passing of some parameters to restrict what edges get swung over:
+ * > otherEndNodeTypeRestriction: only swing edges that terminate on the oldNode if the
+ * node at the other end of the edge is of this nodeType.
+ * > edgeLabelRestriction: Only swing edges that have this edgeLabel
+ * > edgeDirectionRestriction: Only swing edges that go this direction (from the oldNode)
+ * this is a required parameter. valid values are: BOTH, IN, OUT
+ *
+ */
+@MigrationPriority(0)
+@MigrationDangerRating(1)
+public abstract class EdgeSwingMigrator extends Migrator {
+
+ private boolean success = true;
+ private String nodeTypeRestriction = null;
+ private String edgeLabelRestriction = null;
+ private String edgeDirRestriction = null;
+ private List<Pair<Vertex, Vertex>> nodePairList;
+
+
+ public EdgeSwingMigrator(TransactionalGraphEngine engine , LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+
+ /**
+ * Do not override this method as an inheritor of this class
+ */
+ @Override
+ public void run() {
+ executeModifyOperation();
+ cleanupAsAppropriate(this.nodePairList);
+ }
+
+ /**
+ * This is where inheritors should add their logic
+ */
+ protected void executeModifyOperation() {
+
+ try {
+ this.nodeTypeRestriction = this.getNodeTypeRestriction();
+ this.edgeLabelRestriction = this.getEdgeLabelRestriction();
+ this.edgeDirRestriction = this.getEdgeDirRestriction();
+ nodePairList = this.getAffectedNodePairs();
+ for (Pair<Vertex, Vertex> nodePair : nodePairList) {
+ Vertex fromNode = nodePair.getValue0();
+ Vertex toNode = nodePair.getValue1();
+ this.swingEdges(fromNode, toNode,
+ this.nodeTypeRestriction,this.edgeLabelRestriction,this.edgeDirRestriction);
+ }
+ } catch (Exception e) {
+ logger.error("error encountered", e);
+ success = false;
+ }
+ }
+
+
+ protected void swingEdges(Vertex oldNode, Vertex newNode, String nodeTypeRestr, String edgeLabelRestr, String edgeDirRestr) {
+ try {
+ // If the old and new Vertices aren't populated, throw an exception
+ if( oldNode == null ){
+ logger.info ( "null oldNode passed to swingEdges() ");
+ success = false;
+ return;
+ }
+ else if( newNode == null ){
+ logger.info ( "null newNode passed to swingEdges() ");
+ success = false;
+ return;
+ }
+ else if( edgeDirRestr == null ||
+ (!edgeDirRestr.equals("BOTH")
+ && !edgeDirRestr.equals("IN")
+ && !edgeDirRestr.equals("OUT") )
+ ){
+ logger.info ( "invalid direction passed to swingEdges(). valid values are BOTH/IN/OUT ");
+ success = false;
+ return;
+ }
+ else if( edgeLabelRestr != null
+ && (edgeLabelRestr.trim().equals("none") || edgeLabelRestr.trim().equals("")) ){
+ edgeLabelRestr = null;
+ }
+ else if( nodeTypeRestr == null || nodeTypeRestr.trim().equals("") ){
+ nodeTypeRestr = "none";
+ }
+
+ String oldNodeType = oldNode.value(AAIProperties.NODE_TYPE);
+ String oldUri = oldNode.<String> property("aai-uri").isPresent() ? oldNode.<String> property("aai-uri").value() : "URI Not present";
+
+ String newNodeType = newNode.value(AAIProperties.NODE_TYPE);
+ String newUri = newNode.<String> property("aai-uri").isPresent() ? newNode.<String> property("aai-uri").value() : "URI Not present";
+
+ // If the nodeTypes don't match, throw an error
+ if( !oldNodeType.equals(newNodeType) ){
+ logger.info ( "Can not swing edge from a [" + oldNodeType + "] node to a [" +
+ newNodeType + "] node. ");
+ success = false;
+ return;
+ }
+
+ // Find and migrate any applicable OUT edges.
+ if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("OUT") ){
+ Iterator <Edge> edgeOutIter = null;
+ if( edgeLabelRestr == null ) {
+ edgeOutIter = oldNode.edges(Direction.OUT);
+ }
+ else {
+ edgeOutIter = oldNode.edges(Direction.OUT, edgeLabelRestr);
+ }
+
+ while( edgeOutIter.hasNext() ){
+ Edge oldOutE = edgeOutIter.next();
+ String eLabel = oldOutE.label();
+ Vertex otherSideNode4ThisEdge = oldOutE.inVertex();
+ String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
+ if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){
+ Iterator <Property<Object>> propsIter = oldOutE.properties();
+ HashMap<String, String> propMap = new HashMap<String,String>();
+ while( propsIter.hasNext() ){
+ Property <Object> ep = propsIter.next();
+ propMap.put(ep.key(), ep.value().toString());
+ }
+
+ String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
+ logger.info ( "\nSwinging [" + eLabel + "] OUT edge. \n >> Unchanged side is ["
+ + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
+ + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
+ // remove the old edge
+ oldOutE.remove();
+
+ // add the new edge with properties that match the edge that was deleted. We don't want to
+ // change any edge properties - just swinging one end of the edge to a new node.
+ // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are
+ // adding from the newNode side.
+ Edge newOutE = newNode.addEdge(eLabel, otherSideNode4ThisEdge);
+
+ Iterator it = propMap.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry pair = (Map.Entry)it.next();
+ newOutE.property(pair.getKey().toString(), pair.getValue().toString() );
+ }
+
+ }
+ }
+ }
+
+ // Find and migrate any applicable IN edges.
+ if( edgeDirRestr.equals("BOTH") || edgeDirRestr.equals("IN") ){
+ Iterator <Edge> edgeInIter = null;
+ if( edgeLabelRestr == null ) {
+ edgeInIter = oldNode.edges(Direction.IN);
+ }
+ else {
+ edgeInIter = oldNode.edges(Direction.IN, edgeLabelRestr);
+ }
+
+ while( edgeInIter.hasNext() ){
+ Edge oldInE = edgeInIter.next();
+ String eLabel = oldInE.label();
+ Vertex otherSideNode4ThisEdge = oldInE.outVertex();
+ String otherSideNodeType = otherSideNode4ThisEdge.value(AAIProperties.NODE_TYPE);
+ if( nodeTypeRestr.equals("none") || nodeTypeRestr.toLowerCase().equals(otherSideNodeType) ){
+ Iterator <Property<Object>> propsIter = oldInE.properties();
+ HashMap<String, String> propMap = new HashMap<String,String>();
+ while( propsIter.hasNext() ){
+ Property <Object> ep = propsIter.next();
+ propMap.put(ep.key(), ep.value().toString());
+ }
+
+ String otherSideUri = otherSideNode4ThisEdge.<String> property("aai-uri").isPresent() ? otherSideNode4ThisEdge.<String> property("aai-uri").value() : "URI Not present";
+ logger.info ( "\nSwinging [" + eLabel + "] IN edge. \n >> Unchanged side is ["
+ + otherSideNodeType + "][" + otherSideUri + "] \n >> Edge used to go to [" + oldNodeType
+ + "][" + oldUri + "],\n >> now swung to [" + newNodeType + "][" + newUri + "]. ");
+
+ // remove the old edge
+ oldInE.remove();
+
+ // add the new edge with properties that match the edge that was deleted. We don't want to
+ // change any edge properties - just swinging one end of the edge to a new node.
+ // NOTE - addEdge adds an OUT edge to the vertex passed as a parameter, so we are
+ // adding from the node on the other-end of the original edge so we'll get
+ // an IN-edge to the newNode.
+ Edge newInE = otherSideNode4ThisEdge.addEdge(eLabel, newNode);
+
+ Iterator it = propMap.entrySet().iterator();
+ while (it.hasNext()) {
+ Map.Entry pair = (Map.Entry)it.next();
+ newInE.property(pair.getKey().toString(), pair.getValue().toString() );
+ }
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ logger.error("error encountered", e);
+ success = false;
+ }
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+
+ /**
+ * Get the List of node pairs("from" and "to"), you would like EdgeSwingMigrator to migrate from json files
+ * @return
+ */
+ public abstract List<Pair<Vertex, Vertex>> getAffectedNodePairs() ;
+
+
+ /**
+ * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
+ * @return
+ */
+ public abstract String getNodeTypeRestriction() ;
+
+
+ /**
+ * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
+ * @return
+ */
+ public abstract String getEdgeLabelRestriction() ;
+
+ /**
+ * Get the nodeTypeRestriction that you want EdgeSwingMigrator to use
+ * @return
+ */
+ public abstract String getEdgeDirRestriction() ;
+
+
+
+ /**
+ * Cleanup (remove) the nodes that edges were moved off of if appropriate
+ * @return
+ */
+ public abstract void cleanupAsAppropriate(List<Pair<Vertex, Vertex>> nodePairL);
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/migration/Enabled.java b/src/main/java/org/onap/aai/migration/Enabled.java
new file mode 100644
index 0000000..1b7bba3
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/Enabled.java
@@ -0,0 +1,35 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * Used to enable a migration to be picked up by the {@link org.onap.aai.migration.MigrationControllerInternal MigrationController}
+ */
+@Target(ElementType.TYPE)
+@Retention(value = RetentionPolicy.RUNTIME)
+public @interface Enabled {
+
+}
diff --git a/src/main/java/org/onap/aai/migration/EventAction.java b/src/main/java/org/onap/aai/migration/EventAction.java
new file mode 100644
index 0000000..830685b
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/EventAction.java
@@ -0,0 +1,29 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+/**
+ * Used to describe the type of DMaaP event you would like to create
+ */
+public enum EventAction {
+ CREATE,
+ UPDATE,
+ DELETE
+}
diff --git a/src/main/java/org/onap/aai/migration/MigrationController.java b/src/main/java/org/onap/aai/migration/MigrationController.java
new file mode 100644
index 0000000..0e65745
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/MigrationController.java
@@ -0,0 +1,78 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.util.UUID;
+
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConstants;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+/**
+ * Wrapper class to allow {@link org.onap.aai.migration.MigrationControllerInternal MigrationControllerInternal}
+ * to be run from a shell script
+ */
+public class MigrationController {
+
+ /**
+ * The main method.
+ *
+ * @param args
+ * the arguments
+ */
+ public static void main(String[] args) {
+
+ LoggingContext.init();
+ LoggingContext.partnerName("Migration");
+ LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.component("MigrationController");
+ LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.targetServiceName("main");
+ LoggingContext.requestId(UUID.randomUUID().toString());
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ EdgeIngestor edgeIngestor = ctx.getBean(EdgeIngestor.class);
+ EdgeSerializer edgeSerializer = ctx.getBean(EdgeSerializer.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+
+ MigrationControllerInternal internal = new MigrationControllerInternal(loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+
+ try {
+ internal.run(args);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ AAIGraph.getInstance().graphShutdown();
+ System.exit(0);
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java
new file mode 100644
index 0000000..8ef0603
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/MigrationControllerInternal.java
@@ -0,0 +1,498 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.io.IoCore;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.util.AAIConstants;
+import org.onap.aai.util.FormatDate;
+import org.reflections.Reflections;
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+
+/**
+ * Runs a series of migrations from a defined directory based on the presence of
+ * the {@link org.onap.aai.migration.Enabled Enabled} annotation
+ *
+ * It will also write a record of the migrations run to the database.
+ */
+public class MigrationControllerInternal {
+
+ private EELFLogger logger;
+ private final int DANGER_ZONE = 10;
+ public static final String VERTEX_TYPE = "migration-list-1707";
+ private final List<String> resultsSummary = new ArrayList<>();
+ private final List<NotificationHelper> notifications = new ArrayList<>();
+ private static final String SNAPSHOT_LOCATION = AAIConstants.AAI_HOME + AAIConstants.AAI_FILESEP + "logs" + AAIConstants.AAI_FILESEP + "data" + AAIConstants.AAI_FILESEP + "migrationSnapshots";
+
+ private LoaderFactory loaderFactory;
+ private EdgeIngestor edgeIngestor;
+ private EdgeSerializer edgeSerializer;
+ private final SchemaVersions schemaVersions;
+
+ public MigrationControllerInternal(LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
+ this.loaderFactory = loaderFactory;
+ this.edgeIngestor = edgeIngestor;
+ this.edgeSerializer = edgeSerializer;
+ this.schemaVersions = schemaVersions;
+ }
+
+ /**
+ * The main method.
+ *
+ * @param args
+ * the arguments
+ */
+ public void run(String[] args) {
+ // Set the logging file properties to be used by EELFManager
+ System.setProperty("aai.service.name", MigrationController.class.getSimpleName());
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
+
+ logger = EELFManager.getInstance().getLogger(MigrationControllerInternal.class.getSimpleName());
+ MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
+
+ boolean loadSnapshot = false;
+
+ CommandLineArgs cArgs = new CommandLineArgs();
+
+ JCommander jCommander = new JCommander(cArgs, args);
+ jCommander.setProgramName(MigrationController.class.getSimpleName());
+
+ // Set flag to load from snapshot based on the presence of snapshot and
+ // graph storage backend of inmemory
+ if (cArgs.dataSnapshot != null && !cArgs.dataSnapshot.isEmpty()) {
+ try {
+ PropertiesConfiguration config = new PropertiesConfiguration(cArgs.config);
+ if (config.getString("storage.backend").equals("inmemory")) {
+ loadSnapshot = true;
+ System.setProperty("load.snapshot.file", "true");
+ System.setProperty("snapshot.location", cArgs.dataSnapshot);
+ }
+ } catch (ConfigurationException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("ERROR: Could not load janusgraph configuration.\n" + ExceptionUtils.getFullStackTrace(e));
+ return;
+ }
+ }
+ System.setProperty("realtime.db.config", cArgs.config);
+ logAndPrint("\n\n---------- Connecting to Graph ----------");
+ AAIGraph.getInstance();
+
+ logAndPrint("---------- Connection Established ----------");
+ SchemaVersion version = schemaVersions.getDefaultVersion();
+ QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ ModelType introspectorFactoryType = ModelType.MOXY;
+ Loader loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ TransactionalGraphEngine engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+
+ if (cArgs.help) {
+ jCommander.usage();
+ engine.rollback();
+ return;
+ }
+
+ Reflections reflections = new Reflections("org.onap.aai.migration");
+ List<Class<? extends Migrator>> migratorClasses = new ArrayList<>(findClasses(reflections));
+ //Displays list of migration classes which needs to be executed.Pass flag "-l" following by the class names
+ if (cArgs.list) {
+ listMigrationWithStatus(cArgs, migratorClasses, engine);
+ return;
+ }
+
+ logAndPrint("---------- Looking for migration scripts to be executed. ----------");
+ //Excluding any migration class when run migration from script.Pass flag "-e" following by the class names
+ if (!cArgs.excludeClasses.isEmpty()) {
+ migratorClasses = filterMigrationClasses(cArgs.excludeClasses, migratorClasses);
+ listMigrationWithStatus(cArgs, migratorClasses, engine);
+ }
+ List<Class<? extends Migrator>> migratorClassesToRun = createMigratorList(cArgs, migratorClasses);
+
+ sortList(migratorClassesToRun);
+
+ if (!cArgs.scripts.isEmpty() && migratorClassesToRun.isEmpty()) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logAndPrint("\tERROR: Failed to find migrations " + cArgs.scripts + ".");
+ logAndPrint("---------- Done ----------");
+ LoggingContext.successStatusFields();
+ }
+
+ logAndPrint("\tFound " + migratorClassesToRun.size() + " migration scripts.");
+ logAndPrint("---------- Executing Migration Scripts ----------");
+
+
+ if (!cArgs.skipPreMigrationSnapShot) {
+ takePreSnapshotIfRequired(engine, cArgs, migratorClassesToRun);
+ }
+
+ for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
+ String name = migratorClass.getSimpleName();
+ Migrator migrator;
+ if (migratorClass.isAnnotationPresent(Enabled.class)) {
+
+ try {
+ engine.startTransaction();
+ if (!cArgs.forced && hasAlreadyRun(name, engine)) {
+ logAndPrint("Migration " + name + " has already been run on this database and will not be executed again. Use -f to force execution");
+ continue;
+ }
+ migrator = migratorClass
+ .getConstructor(
+ TransactionalGraphEngine.class,
+ LoaderFactory.class,
+ EdgeIngestor.class,
+ EdgeSerializer.class,
+ SchemaVersions.class
+ ).newInstance(engine, loaderFactory, edgeIngestor, edgeSerializer,schemaVersions);
+ } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("EXCEPTION caught initalizing migration class " + migratorClass.getSimpleName() + ".\n" + ExceptionUtils.getFullStackTrace(e));
+ LoggingContext.successStatusFields();
+ engine.rollback();
+ continue;
+ }
+ logAndPrint("\tRunning " + migratorClass.getSimpleName() + " migration script.");
+ logAndPrint("\t\t See " + System.getProperty("AJSC_HOME") + "/logs/migration/" + migratorClass.getSimpleName() + "/* for logs.");
+ MDC.put("logFilenameAppender", migratorClass.getSimpleName() + "/" + migratorClass.getSimpleName());
+
+ migrator.run();
+
+ commitChanges(engine, migrator, cArgs);
+ } else {
+ logAndPrint("\tSkipping " + migratorClass.getSimpleName() + " migration script because it has been disabled.");
+ }
+ }
+ MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
+ for (NotificationHelper notificationHelper : notifications) {
+ try {
+ notificationHelper.triggerEvents();
+ } catch (AAIException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint("\tcould not event");
+ logger.error("could not event", e);
+ LoggingContext.successStatusFields();
+ }
+ }
+ logAndPrint("---------- Done ----------");
+
+ // Save post migration snapshot if snapshot was loaded
+ if (!cArgs.skipPostMigrationSnapShot) {
+ generateSnapshot(engine, "post");
+ }
+
+ outputResultsSummary();
+ }
+
+ /**
+ * This method is used to remove excluded classes from migration from the
+ * script command.
+ *
+ * @param excludeClasses
+ * : Classes to be removed from Migration
+ * @param migratorClasses
+ * : Classes to execute migration.
+ * @return
+ */
+ private List<Class<? extends Migrator>> filterMigrationClasses(
+ List<String> excludeClasses,
+ List<Class<? extends Migrator>> migratorClasses) {
+
+ List<Class<? extends Migrator>> filteredMigratorClasses = migratorClasses
+ .stream()
+ .filter(migratorClass -> !excludeClasses.contains(migratorClass
+ .getSimpleName())).collect(Collectors.toList());
+
+ return filteredMigratorClasses;
+ }
+
+ private void listMigrationWithStatus(CommandLineArgs cArgs,
+ List<Class<? extends Migrator>> migratorClasses, TransactionalGraphEngine engine) {
+ sortList(migratorClasses);
+ engine.startTransaction();
+ System.out.println("---------- List of all migrations ----------");
+ migratorClasses.forEach(migratorClass -> {
+ boolean enabledAnnotation = migratorClass.isAnnotationPresent(Enabled.class);
+ String enabled = enabledAnnotation ? "Enabled" : "Disabled";
+ StringBuilder sb = new StringBuilder();
+ sb.append(migratorClass.getSimpleName());
+ sb.append(" in package ");
+ sb.append(migratorClass.getPackage().getName().substring(migratorClass.getPackage().getName().lastIndexOf('.')+1));
+ sb.append(" is ");
+ sb.append(enabled);
+ sb.append(" ");
+ sb.append("[" + getDbStatus(migratorClass.getSimpleName(), engine) + "]");
+ System.out.println(sb.toString());
+ });
+ engine.rollback();
+ System.out.println("---------- Done ----------");
+ }
+
+ private String getDbStatus(String name, TransactionalGraphEngine engine) {
+ if (hasAlreadyRun(name, engine)) {
+ return "Already executed in this env";
+ }
+ return "Will be run on next execution if Enabled";
+ }
+
+ private boolean hasAlreadyRun(String name, TransactionalGraphEngine engine) {
+ return engine.asAdmin().getReadOnlyTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).has(name, true).hasNext();
+ }
+ private Set<Class<? extends Migrator>> findClasses(Reflections reflections) {
+ Set<Class<? extends Migrator>> migratorClasses = reflections.getSubTypesOf(Migrator.class);
+ /*
+ * TODO- Change this to make sure only classes in the specific $release are added in the runList
+ * Or add a annotation like exclude which folks again need to remember to add ??
+ */
+
+ migratorClasses.remove(PropertyMigrator.class);
+ migratorClasses.remove(EdgeMigrator.class);
+ return migratorClasses;
+ }
+
+
+ private void takePreSnapshotIfRequired(TransactionalGraphEngine engine, CommandLineArgs cArgs, List<Class<? extends Migrator>> migratorClassesToRun) {
+
+ /*int sum = 0;
+ for (Class<? extends Migrator> migratorClass : migratorClassesToRun) {
+ if (migratorClass.isAnnotationPresent(Enabled.class)) {
+ sum += migratorClass.getAnnotation(MigrationPriority.class).value();
+ }
+ }
+
+ if (sum >= DANGER_ZONE) {
+
+ logAndPrint("Entered Danger Zone. Taking snapshot.");
+ }*/
+
+ //always take snapshot for now
+
+ generateSnapshot(engine, "pre");
+
+ }
+
+
+ private List<Class<? extends Migrator>> createMigratorList(CommandLineArgs cArgs,
+ List<Class<? extends Migrator>> migratorClasses) {
+ List<Class<? extends Migrator>> migratorClassesToRun = new ArrayList<>();
+
+ for (Class<? extends Migrator> migratorClass : migratorClasses) {
+ if (!cArgs.scripts.isEmpty() && !cArgs.scripts.contains(migratorClass.getSimpleName())) {
+ continue;
+ } else {
+ migratorClassesToRun.add(migratorClass);
+ }
+ }
+ return migratorClassesToRun;
+ }
+
+
+ private void sortList(List<Class<? extends Migrator>> migratorClasses) {
+ Collections.sort(migratorClasses, (m1, m2) -> {
+ try {
+ if (m1.getAnnotation(MigrationPriority.class).value() > m2.getAnnotation(MigrationPriority.class).value()) {
+ return 1;
+ } else if (m1.getAnnotation(MigrationPriority.class).value() < m2.getAnnotation(MigrationPriority.class).value()) {
+ return -1;
+ } else {
+ return m1.getSimpleName().compareTo(m2.getSimpleName());
+ }
+ } catch (Exception e) {
+ return 0;
+ }
+ });
+ }
+
+
+ private void generateSnapshot(TransactionalGraphEngine engine, String phase) {
+
+ FormatDate fd = new FormatDate("yyyyMMddHHmm", "GMT");
+ String dateStr= fd.getDateTime();
+ String fileName = SNAPSHOT_LOCATION + File.separator + phase + "Migration." + dateStr + ".graphson";
+ logAndPrint("Saving snapshot of graph " + phase + " migration to " + fileName);
+ Graph transaction = null;
+ try {
+
+ Path pathToFile = Paths.get(fileName);
+ if (!pathToFile.toFile().exists()) {
+ Files.createDirectories(pathToFile.getParent());
+ }
+ transaction = engine.startTransaction();
+ transaction.io(IoCore.graphson()).writeGraph(fileName);
+ engine.rollback();
+ } catch (IOException e) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint("ERROR: Could not write in memory graph to " + phase + "Migration file. \n" + ExceptionUtils.getFullStackTrace(e));
+ LoggingContext.successStatusFields();
+ engine.rollback();
+ }
+
+ logAndPrint( phase + " migration snapshot saved to " + fileName);
+ }
+ /**
+ * Log and print.
+ *
+ * @param msg
+ * the msg
+ */
+ protected void logAndPrint(String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+ /**
+ * Commit changes.
+ *
+ * @param engine
+ * the graph transaction
+ * @param migrator
+ * the migrator
+ * @param cArgs
+ */
+ protected void commitChanges(TransactionalGraphEngine engine, Migrator migrator, CommandLineArgs cArgs) {
+
+ String simpleName = migrator.getClass().getSimpleName();
+ String message;
+ if (migrator.getStatus().equals(Status.FAILURE)) {
+ message = "Migration " + simpleName + " Failed. Rolling back.";
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("\t" + message);
+ LoggingContext.successStatusFields();
+ migrator.rollback();
+ } else if (migrator.getStatus().equals(Status.CHECK_LOGS)) {
+ message = "Migration " + simpleName + " encountered an anomaly, check logs. Rolling back.";
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint("\t" + message);
+ LoggingContext.successStatusFields();
+ migrator.rollback();
+ } else {
+ MDC.put("logFilenameAppender", simpleName + "/" + simpleName);
+
+ if (cArgs.commit) {
+ if (!engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE).hasNext()) {
+ engine.asAdmin().getTraversalSource().addV(AAIProperties.NODE_TYPE, VERTEX_TYPE).iterate();
+ }
+ engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VERTEX_TYPE)
+ .property(simpleName, true).iterate();
+ MDC.put("logFilenameAppender", MigrationController.class.getSimpleName());
+ notifications.add(migrator.getNotificationHelper());
+ migrator.commit();
+ message = "Migration " + simpleName + " Succeeded. Changes Committed.";
+ logAndPrint("\t"+ message +"\t");
+ } else {
+ message = "--commit not specified. Not committing changes for " + simpleName + " to database.";
+ logAndPrint("\t" + message);
+ migrator.rollback();
+ }
+
+ }
+
+ resultsSummary.add(message);
+
+ }
+
+ private void outputResultsSummary() {
+ logAndPrint("---------------------------------");
+ logAndPrint("-------------Summary-------------");
+ for (String result : resultsSummary) {
+ logAndPrint(result);
+ }
+ logAndPrint("---------------------------------");
+ logAndPrint("---------------------------------");
+ }
+
+}
+
+class CommandLineArgs {
+
+ @Parameter(names = "--help", help = true)
+ public boolean help;
+
+ @Parameter(names = "-c", description = "location of configuration file")
+ public String config;
+
+ @Parameter(names = "-m", description = "names of migration scripts")
+ public List<String> scripts = new ArrayList<>();
+
+ @Parameter(names = "-l", description = "list the status of migrations")
+ public boolean list = false;
+
+ @Parameter(names = "-d", description = "location of data snapshot", hidden = true)
+ public String dataSnapshot;
+
+ @Parameter(names = "-f", description = "force migrations to be rerun")
+ public boolean forced = false;
+
+ @Parameter(names = "--commit", description = "commit changes to graph")
+ public boolean commit = false;
+
+ @Parameter(names = "-e", description = "exclude list of migrator classes")
+ public List<String> excludeClasses = new ArrayList<>();
+
+ @Parameter(names = "--skipPreMigrationSnapShot", description = "skips taking the PRE migration snapshot")
+ public boolean skipPreMigrationSnapShot = false;
+
+ @Parameter(names = "--skipPostMigrationSnapShot", description = "skips taking the POST migration snapshot")
+ public boolean skipPostMigrationSnapShot = false;
+}
diff --git a/src/main/java/org/onap/aai/migration/MigrationDangerRating.java b/src/main/java/org/onap/aai/migration/MigrationDangerRating.java
new file mode 100644
index 0000000..1d82dc3
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/MigrationDangerRating.java
@@ -0,0 +1,41 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * Used to enable a migration to be picked up by the {@link com.openecomp.aai.migration.MigrationControllerInternal MigrationController}
+ *
+ * The larger the number, the more danger
+ *
+ * Range is 0-10
+ */
+@Target(ElementType.TYPE)
+@Retention(value = RetentionPolicy.RUNTIME)
+public @interface MigrationDangerRating {
+
+ int value();
+
+}
diff --git a/src/main/java/org/onap/aai/migration/MigrationPriority.java b/src/main/java/org/onap/aai/migration/MigrationPriority.java
new file mode 100644
index 0000000..d9e84b8
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/MigrationPriority.java
@@ -0,0 +1,41 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+
+/**
+ * Used to enable a migration to be picked up by the {@link org.onap.aai.migration.MigrationControllerInternal MigrationController}
+ *
+ * The priority of the migration.
+ *
+ * Lower number has higher priority
+ */
+@Target(ElementType.TYPE)
+@Retention(value = RetentionPolicy.RUNTIME)
+public @interface MigrationPriority {
+
+ int value();
+
+}
diff --git a/src/main/java/org/onap/aai/migration/Migrator.java b/src/main/java/org/onap/aai/migration/Migrator.java
new file mode 100644
index 0000000..106d5e4
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/Migrator.java
@@ -0,0 +1,385 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException;
+import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+/**
+ * This class defines an A&AI Migration
+ */
+@MigrationPriority(0)
+@MigrationDangerRating(0)
+public abstract class Migrator implements Runnable {
+
+ protected EELFLogger logger = null;
+
+ protected DBSerializer serializer = null;
+ protected Loader loader = null;
+
+ protected TransactionalGraphEngine engine;
+ protected NotificationHelper notificationHelper;
+
+ protected EdgeSerializer edgeSerializer;
+ protected EdgeIngestor edgeIngestor;
+
+ protected LoaderFactory loaderFactory;
+ protected SchemaVersions schemaVersions;
+
+ protected static final String MIGRATION_ERROR = "Migration Error: ";
+ protected static final String MIGRATION_SUMMARY_COUNT = "Migration Summary Count: ";
+
+ /**
+ * Instantiates a new migrator.
+ *
+ * @param g the g
+ * @param schemaVersions
+ */
+ public Migrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
+ this.engine = engine;
+ this.loaderFactory = loaderFactory;
+ this.edgeIngestor = edgeIngestor;
+ this.edgeSerializer = edgeSerializer;
+ this.schemaVersions = schemaVersions;
+ initDBSerializer();
+ this.notificationHelper = new NotificationHelper(loader, serializer, loaderFactory, schemaVersions, engine, "AAI-MIGRATION", this.getMigrationName());
+ logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName());
+ logger.info("\tInitilization of " + this.getClass().getSimpleName() + " migration script complete.");
+ }
+
+ /**
+ * Gets the status.
+ *
+ * @return the status
+ */
+ public abstract Status getStatus();
+
+ /**
+ * Rollback.
+ */
+ public void rollback() {
+ engine.rollback();
+ }
+
+ /**
+ * Commit.
+ */
+ public void commit() {
+ engine.commit();
+ }
+
+ /**
+ * Create files containing vertices for dmaap Event Generation
+ * @param dmaapMsgList
+ */
+ public void createDmaapFiles(List<String> dmaapMsgList) {
+ String fileName = getMigrationName() + "-" + UUID.randomUUID();
+ String logDirectory = System.getProperty("AJSC_HOME") + "/logs/migration/dmaapEvents";
+
+ File f = new File(logDirectory);
+ f.mkdirs();
+
+ if (dmaapMsgList.size() > 0) {
+ try {
+ Files.write(Paths.get(logDirectory+"/"+fileName), (Iterable<String>)dmaapMsgList.stream()::iterator);
+ } catch (IOException e) {
+ logger.error("Unable to generate file with dmaap msgs for MigrateHUBEvcInventory", e);
+ }
+ } else {
+ logger.info("No dmaap msgs detected for MigrateForwardEvcCircuitId");
+ }
+ }
+
+ /**
+ * Create files containing data for dmaap delete Event Generation
+ * @param dmaapVertexList
+ */
+ public void createDmaapFilesForDelete(List<Introspector> dmaapDeleteIntrospectorList) {try {
+ System.out.println("dmaapDeleteIntrospectorList :: " + dmaapDeleteIntrospectorList.size());
+ String fileName = "DELETE-"+ getMigrationName() + "-" + UUID.randomUUID();
+ String logDirectory = System.getProperty("AJSC_HOME") + "/logs/migration/dmaapEvents/";
+ File f = new File(logDirectory);
+ f.mkdirs();
+
+ try{
+ Files.createFile(Paths.get(logDirectory + "/" + fileName));
+ }catch(Exception e) {
+ e.printStackTrace();
+ }
+
+ if (dmaapDeleteIntrospectorList.size() > 0) {
+ dmaapDeleteIntrospectorList.stream().forEach(svIntr-> {
+ try {
+ String str = svIntr.marshal(false);
+ String finalStr="";
+ try {
+ finalStr=svIntr.getName() + "#@#" + svIntr.getURI() + "#@#" + str+"\n";
+ Files.write(Paths.get(logDirectory + "/" + fileName),finalStr.getBytes(),StandardOpenOption.APPEND);
+ } catch (IOException e) {
+ logger.error("Unable to generate file with dmaap msgs for "+getMigrationName(), e);
+ }
+
+ }catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ });
+
+ //Files.write(Paths.get(logDirectory+"/"+fileName), (Iterable<Vertex>)dmaapVertexList.stream()::iterator);
+ }
+ }catch (Exception e) {
+ e.printStackTrace();
+ logger.error("Unable to generate file with dmaap msgs for "+getMigrationName(), e);
+ }}
+
+ /**
+ * As string.
+ *
+ * @param v the v
+ * @return the string
+ */
+ protected String asString(Vertex v) {
+ final JSONObject result = new JSONObject();
+ Iterator<VertexProperty<Object>> properties = v.properties();
+ Property<Object> pk = null;
+ try {
+ while (properties.hasNext()) {
+ pk = properties.next();
+ result.put(pk.key(), pk.value());
+ }
+ } catch (JSONException e) {
+ logger.error("Warning error reading vertex: " + e);
+ }
+
+ return result.toString();
+ }
+
+ /**
+ * As string.
+ *
+ * @param edge the edge
+ * @return the string
+ */
+ protected String asString(Edge edge) {
+ final JSONObject result = new JSONObject();
+ Iterator<Property<Object>> properties = edge.properties();
+ Property<Object> pk = null;
+ try {
+ while (properties.hasNext()) {
+ pk = properties.next();
+ result.put(pk.key(), pk.value());
+ }
+ } catch (JSONException e) {
+ logger.error("Warning error reading edge: " + e);
+ }
+
+ return result.toString();
+ }
+
+ /**
+ *
+ * @param v
+ * @param numLeadingTabs number of leading \t char's
+ * @return
+ */
+ protected String toStringForPrinting(Vertex v, int numLeadingTabs) {
+ String prefix = String.join("", Collections.nCopies(numLeadingTabs, "\t"));
+ if (v == null) {
+ return "";
+ }
+ final StringBuilder sb = new StringBuilder();
+ sb.append(prefix + v + "\n");
+ v.properties().forEachRemaining(prop -> sb.append(prefix + prop + "\n"));
+ return sb.toString();
+ }
+
+ /**
+ *
+ * @param e
+ * @param numLeadingTabs number of leading \t char's
+ * @return
+ */
+ protected String toStringForPrinting(Edge e, int numLeadingTabs) {
+ String prefix = String.join("", Collections.nCopies(numLeadingTabs, "\t"));
+ if (e == null) {
+ return "";
+ }
+ final StringBuilder sb = new StringBuilder();
+ sb.append(prefix + e + "\n");
+ sb.append(prefix + e.label() + "\n");
+ e.properties().forEachRemaining(prop -> sb.append(prefix + "\t" + prop + "\n"));
+ return sb.toString();
+ }
+
+ /**
+ * Checks for edge between.
+ *
+ * @param a a
+ * @param b b
+ * @param d d
+ * @param edgeLabel the edge label
+ * @return true, if successful
+ */
+ protected boolean hasEdgeBetween(Vertex a, Vertex b, Direction d, String edgeLabel) {
+
+ if (d.equals(Direction.OUT)) {
+ return engine.asAdmin().getReadOnlyTraversalSource().V(a).out(edgeLabel).where(__.otherV().hasId(b)).hasNext();
+ } else {
+ return engine.asAdmin().getReadOnlyTraversalSource().V(a).in(edgeLabel).where(__.otherV().hasId(b)).hasNext();
+ }
+
+ }
+
+ /**
+ * Creates the edge
+ *
+ * @param type the edge type - COUSIN or TREE
+ * @param out the out
+ * @param in the in
+ * @return the edge
+ */
+ protected Edge createEdge(EdgeType type, Vertex out, Vertex in) throws AAIException {
+ Edge newEdge = null;
+ try {
+ if (type.equals(EdgeType.COUSIN)){
+ newEdge = edgeSerializer.addEdge(this.engine.asAdmin().getTraversalSource(), out, in);
+ } else {
+ newEdge = edgeSerializer.addTreeEdge(this.engine.asAdmin().getTraversalSource(), out, in);
+ }
+ } catch (NoEdgeRuleFoundException e) {
+ throw new AAIException("AAI_6129", e);
+ }
+ return newEdge;
+ }
+
+ /**
+ * Creates the edge
+ *
+ * @param type the edge type - COUSIN or TREE
+ * @param out the out
+ * @param in the in
+ * @return the edge
+ */
+ protected Edge createPrivateEdge(Vertex out, Vertex in) throws AAIException {
+ Edge newEdge = null;
+ try {
+ newEdge = edgeSerializer.addPrivateEdge(this.engine.asAdmin().getTraversalSource(), out, in, null);
+ } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
+ throw new AAIException("AAI_6129", e);
+ }
+ return newEdge;
+ }
+
+ /**
+ * Creates the TREE edge
+ *
+ * @param out the out
+ * @param in the in
+ * @return the edge
+ */
+ protected Edge createTreeEdge(Vertex out, Vertex in) throws AAIException {
+ Edge newEdge = createEdge(EdgeType.TREE, out, in);
+ return newEdge;
+ }
+
+ /**
+ * Creates the COUSIN edge
+ *
+ * @param out the out
+ * @param in the in
+ * @return the edge
+ */
+ protected Edge createCousinEdge(Vertex out, Vertex in) throws AAIException {
+ Edge newEdge = createEdge(EdgeType.COUSIN, out, in);
+ return newEdge;
+ }
+
+ private void initDBSerializer() {
+ SchemaVersion version = schemaVersions.getDefaultVersion();
+ ModelType introspectorFactoryType = ModelType.MOXY;
+ loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ try {
+ this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.getMigrationName());
+ } catch (AAIException e) {
+ throw new RuntimeException("could not create seralizer", e);
+ }
+ }
+
+ /**
+ * These are the node types you would like your traversal to process
+ * @return
+ */
+ public abstract Optional<String[]> getAffectedNodeTypes();
+
+ /**
+ * used as the "fromAppId" when modifying vertices
+ * @return
+ */
+ public abstract String getMigrationName();
+
+ /**
+ * updates all internal vertex properties
+ * @param v
+ * @param isNewVertex
+ */
+ protected void touchVertexProperties(Vertex v, boolean isNewVertex) {
+ this.serializer.touchStandardVertexProperties(v, isNewVertex);
+ }
+
+ public NotificationHelper getNotificationHelper() {
+ return this.notificationHelper;
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/NotificationHelper.java b/src/main/java/org/onap/aai/migration/NotificationHelper.java
new file mode 100644
index 0000000..ff5c030
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/NotificationHelper.java
@@ -0,0 +1,118 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.util.HashMap;
+import java.util.List;
+
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.rest.ueb.UEBNotification;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.serialization.engines.query.QueryEngine;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.onap.aai.setup.SchemaVersions;
+
+/**
+ * Allows for DMaaP notifications from Migrations
+ */
+public class NotificationHelper {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(NotificationHelper.class);
+ protected final DBSerializer serializer;
+ protected final Loader loader;
+ protected final TransactionalGraphEngine engine;
+ protected final String transactionId;
+ protected final String sourceOfTruth;
+ protected final UEBNotification notification;
+
+ public NotificationHelper(Loader loader, DBSerializer serializer, LoaderFactory loaderFactory, SchemaVersions schemaVersions, TransactionalGraphEngine engine, String transactionId, String sourceOfTruth) {
+ this.loader = loader;
+ this.serializer = serializer;
+ this.engine = engine;
+ this.transactionId = transactionId;
+ this.sourceOfTruth = sourceOfTruth;
+ this.notification = new UEBNotification(loader, loaderFactory, schemaVersions);
+ }
+
+ public void addEvent(Vertex v, Introspector obj, EventAction action, URI uri, String basePath) throws UnsupportedEncodingException, AAIException {
+ HashMap<String, Introspector> relatedObjects = new HashMap<>();
+ Status status = mapAction(action);
+
+ if (!obj.isTopLevel()) {
+ relatedObjects = this.getRelatedObjects(serializer, engine.getQueryEngine(), v);
+ }
+ notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects, basePath);
+
+ }
+
+ public void addDeleteEvent(String transactionId, String sourceOfTruth, EventAction action, URI uri, Introspector obj, HashMap relatedObjects,String basePath) throws UnsupportedEncodingException, AAIException {
+ Status status = mapAction(action);
+ notification.createNotificationEvent(transactionId, sourceOfTruth, status, uri, obj, relatedObjects, basePath);
+
+ }
+
+ private HashMap<String, Introspector> getRelatedObjects(DBSerializer serializer, QueryEngine queryEngine, Vertex v) throws AAIException {
+ HashMap<String, Introspector> relatedVertices = new HashMap<>();
+ List<Vertex> vertexChain = queryEngine.findParents(v);
+ for (Vertex vertex : vertexChain) {
+ try {
+ final Introspector vertexObj = serializer.getVertexProperties(vertex);
+ relatedVertices.put(vertexObj.getObjectId(), vertexObj);
+ } catch (AAIUnknownObjectException | UnsupportedEncodingException e) {
+ LOGGER.warn("Unable to get vertex properties, partial list of related vertices returned");
+ }
+
+ }
+
+ return relatedVertices;
+ }
+
+ private Status mapAction(EventAction action) {
+ if (EventAction.CREATE.equals(action)) {
+ return Status.CREATED;
+ } else if (EventAction.UPDATE.equals(action)) {
+ return Status.OK;
+ } else if (EventAction.DELETE.equals(action)) {
+ return Status.NO_CONTENT;
+ } else {
+ return Status.OK;
+ }
+ }
+
+ public void triggerEvents() throws AAIException {
+ notification.triggerEvents();
+ }
+
+ public UEBNotification getNotifications() {
+ return this.notification;
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/PropertyMigrator.java b/src/main/java/org/onap/aai/migration/PropertyMigrator.java
new file mode 100644
index 0000000..4599243
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/PropertyMigrator.java
@@ -0,0 +1,146 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.util.Optional;
+
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.onap.aai.setup.SchemaVersions;
+
+/**
+ * A migration template for migrating a property from one name to another
+ */
+@MigrationPriority(0)
+@MigrationDangerRating(1)
+public abstract class PropertyMigrator extends Migrator {
+
+ protected String OLD_FIELD;
+ protected String NEW_FIELD;
+ protected Integer changedVertexCount;
+ protected Class<?> fieldType;
+ protected Cardinality cardinality;
+ protected final JanusGraphManagement graphMgmt;
+
+
+ public PropertyMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.changedVertexCount = 0;
+ this.graphMgmt = engine.asAdmin().getManagementSystem();
+ }
+
+ public void initialize(String oldName, String newName, Class<?> type, Cardinality cardinality){
+ this.OLD_FIELD = oldName;
+ this.NEW_FIELD = newName;
+ this.fieldType = type;
+ this.cardinality = cardinality;
+ }
+
+ /**
+ * Do not override this method as an inheritor of this class
+ */
+ @Override
+ public void run() {
+ logger.info("-------- Starting PropertyMigrator for node type " + P.within(this.getAffectedNodeTypes().get())
+ + " from property " + OLD_FIELD + " to " + NEW_FIELD + " --------");
+ modifySchema();
+ executeModifyOperation();
+ logger.info(Migrator.MIGRATION_SUMMARY_COUNT + changedVertexCount + " vertices modified.");
+ }
+
+ protected void modifySchema() {
+ this.addIndex(this.addProperty());
+ graphMgmt.commit();
+ }
+
+ /**
+ * This is where inheritors should add their logic
+ */
+ protected void executeModifyOperation() {
+ changePropertyName();
+ }
+
+ protected void changePropertyName() {
+ GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V();
+ if (this.getAffectedNodeTypes().isPresent()) {
+ g.has(AAIProperties.NODE_TYPE, P.within(this.getAffectedNodeTypes().get()));
+ }
+ g.has(OLD_FIELD).sideEffect(t -> {
+ final Vertex v = t.get();
+ logger.info("Migrating property for vertex " + v.toString());
+ final String value = v.value(OLD_FIELD);
+ v.property(OLD_FIELD).remove();
+ v.property(NEW_FIELD, value);
+ this.touchVertexProperties(v, false);
+ this.changedVertexCount += 1;
+ logger.info(v.toString() + " : Migrated property " + OLD_FIELD + " to " + NEW_FIELD + " with value = " + value);
+ }).iterate();
+ }
+
+ @Override
+ public Status getStatus() {
+ GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V();
+ if (this.getAffectedNodeTypes().isPresent()) {
+ g.has(AAIProperties.NODE_TYPE, P.within(this.getAffectedNodeTypes().get()));
+ }
+ long result = g.has(OLD_FIELD).count().next();
+ if (result == 0) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ protected Optional<PropertyKey> addProperty() {
+
+ if (!graphMgmt.containsPropertyKey(this.NEW_FIELD)) {
+ logger.info(" PropertyKey [" + this.NEW_FIELD + "] created in the DB. ");
+ return Optional.of(graphMgmt.makePropertyKey(this.NEW_FIELD).dataType(this.fieldType).cardinality(this.cardinality)
+ .make());
+ } else {
+ logger.info(" PropertyKey [" + this.NEW_FIELD + "] already existed in the DB. ");
+ return Optional.empty();
+ }
+
+ }
+
+ protected void addIndex(Optional<PropertyKey> key) {
+ if (isIndexed() && key.isPresent()) {
+ if (graphMgmt.containsGraphIndex(key.get().name())) {
+ logger.debug(" Index [" + key.get().name() + "] already existed in the DB. ");
+ } else {
+ logger.info("Add index for PropertyKey: [" + key.get().name() + "]");
+ graphMgmt.buildIndex(key.get().name(), Vertex.class).addKey(key.get()).buildCompositeIndex();
+ }
+ }
+ }
+ public abstract boolean isIndexed();
+
+}
diff --git a/src/main/java/org/onap/aai/migration/Status.java b/src/main/java/org/onap/aai/migration/Status.java
new file mode 100644
index 0000000..0338594
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/Status.java
@@ -0,0 +1,29 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+/**
+ * Defines the status of the completed migration
+ */
+public enum Status {
+ SUCCESS,
+ CHECK_LOGS,
+ FAILURE
+}
diff --git a/src/main/java/org/onap/aai/migration/ValueMigrator.java b/src/main/java/org/onap/aai/migration/ValueMigrator.java
new file mode 100644
index 0000000..6d02563
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/ValueMigrator.java
@@ -0,0 +1,104 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.util.Map;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.onap.aai.setup.SchemaVersions;
+
+/**
+ * A migration template for filling in default values that are missing or are empty
+ */
+@MigrationPriority(0)
+@MigrationDangerRating(1)
+public abstract class ValueMigrator extends Migrator {
+
+ protected final Map<String, Map<String, ?>> propertyValuePairByNodeType;
+ protected final Boolean updateExistingValues;
+ protected final JanusGraphManagement graphMgmt;
+
+ /**
+ *
+ * @param engine
+ * @param propertyValuePairByNodeType - format {nodeType: { property: newValue}}
+ * @param updateExistingValues - if true, updates the value regardless if it already exists
+ */
+ public ValueMigrator(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, Map propertyValuePairByNodeType, Boolean updateExistingValues) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.propertyValuePairByNodeType = propertyValuePairByNodeType;
+ this.updateExistingValues = updateExistingValues;
+ this.graphMgmt = engine.asAdmin().getManagementSystem();
+ }
+
+ /**
+ * Do not override this method as an inheritor of this class
+ */
+ @Override
+ public void run() {
+ updateValues();
+ }
+
+ protected void updateValues() {
+ for (Map.Entry<String, Map<String, ?>> entry: propertyValuePairByNodeType.entrySet()) {
+ String nodeType = entry.getKey();
+ Map<String, ?> propertyValuePair = entry.getValue();
+ for (Map.Entry<String, ?> pair : propertyValuePair.entrySet()) {
+ String property = pair.getKey();
+ Object newValue = pair.getValue();
+ try {
+ GraphTraversal<Vertex, Vertex> g = this.engine.asAdmin().getTraversalSource().V()
+ .has(AAIProperties.NODE_TYPE, nodeType);
+ while (g.hasNext()) {
+ Vertex v = g.next();
+ if (v.property(property).isPresent() && !updateExistingValues) {
+ String propertyValue = v.property(property).value().toString();
+ if (propertyValue.isEmpty()) {
+ v.property(property, newValue);
+ logger.info(String.format("Node Type %s: Property %s is empty, adding value %s",
+ nodeType, property, newValue.toString()));
+ this.touchVertexProperties(v, false);
+ } else {
+ logger.info(String.format("Node Type %s: Property %s value already exists - skipping",
+ nodeType, property));
+ }
+ } else {
+ logger.info(String.format("Node Type %s: Property %s does not exist or " +
+ "updateExistingValues flag is set to True - adding the property with value %s",
+ nodeType, property, newValue.toString()));
+ v.property(property, newValue);
+ this.touchVertexProperties(v, false);
+ }
+ }
+ } catch (Exception e) {
+ logger.error(String.format("caught exception updating aai-node-type %s's property %s's value to " +
+ "%s: %s", nodeType, property, newValue.toString(), e.getMessage()));
+ logger.error(e.getMessage());
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/VertexMerge.java b/src/main/java/org/onap/aai/migration/VertexMerge.java
new file mode 100644
index 0000000..abf19be
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/VertexMerge.java
@@ -0,0 +1,255 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+/**
+ * This class recursively merges two vertices passed in.
+ * <br>
+ * You can start with any two vertices, but after the vertices are merged based off the equality of their keys
+ *
+ */
+public class VertexMerge {
+
+ private final EELFLogger logger = EELFManager.getInstance().getLogger(this.getClass().getSimpleName());
+
+ private final GraphTraversalSource g;
+ private final TransactionalGraphEngine engine;
+ private final DBSerializer serializer;
+ private final EdgeSerializer edgeSerializer;
+ private final Loader loader;
+ private final NotificationHelper notificationHelper;
+ private final boolean hasNotifications;
+
+ private VertexMerge(Builder builder) {
+ this.engine = builder.getEngine();
+ this.serializer = builder.getSerializer();
+ this.g = engine.asAdmin().getTraversalSource();
+ this.edgeSerializer = builder.getEdgeSerializer();
+ this.loader = builder.getLoader();
+ this.notificationHelper = builder.getHelper();
+ this.hasNotifications = builder.isHasNotifications();
+ }
+
+ /**
+ * Merges vertices. forceCopy is a map of the form [{aai-node-type}:{set of properties}]
+ * @param primary
+ * @param secondary
+ * @param forceCopy
+ * @throws AAIException
+ * @throws UnsupportedEncodingException
+ */
+ public void performMerge(Vertex primary, Vertex secondary, Map<String, Set<String>> forceCopy, String basePath) throws AAIException, UnsupportedEncodingException {
+ final Optional<Introspector> secondarySnapshot;
+ if (this.hasNotifications) {
+ secondarySnapshot = Optional.of(serializer.getLatestVersionView(secondary));
+ } else {
+ secondarySnapshot = Optional.empty();
+ }
+ mergeProperties(primary, secondary, forceCopy);
+
+ Collection<Vertex> secondaryChildren = this.engine.getQueryEngine().findChildren(secondary);
+ Collection<Vertex> primaryChildren = this.engine.getQueryEngine().findChildren(primary);
+
+ mergeChildren(primary, secondary, primaryChildren, secondaryChildren, forceCopy);
+
+ Collection<Vertex> secondaryCousins = this.engine.getQueryEngine().findCousinVertices(secondary);
+ Collection<Vertex> primaryCousins = this.engine.getQueryEngine().findCousinVertices(primary);
+
+ secondaryCousins.removeAll(primaryCousins);
+ logger.info("removing vertex after merge: " + secondary );
+ if (this.hasNotifications && secondarySnapshot.isPresent()) {
+ this.notificationHelper.addEvent(secondary, secondarySnapshot.get(), EventAction.DELETE, this.serializer.getURIForVertex(secondary, false), basePath);
+ }
+ secondary.remove();
+ for (Vertex v : secondaryCousins) {
+ this.edgeSerializer.addEdgeIfPossible(g, v, primary);
+ }
+ if (this.hasNotifications) {
+ final Introspector primarySnapshot = serializer.getLatestVersionView(primary);
+ this.notificationHelper.addEvent(primary, primarySnapshot, EventAction.UPDATE, this.serializer.getURIForVertex(primary, false), basePath);
+ }
+ }
+
+ /**
+ * This method may go away if we choose to event on each modification performed
+ * @param primary
+ * @param secondary
+ * @param forceCopy
+ * @throws AAIException
+ * @throws UnsupportedEncodingException
+ */
+ protected void performMergeHelper(Vertex primary, Vertex secondary, Map<String, Set<String>> forceCopy) throws AAIException, UnsupportedEncodingException {
+ mergeProperties(primary, secondary, forceCopy);
+
+ Collection<Vertex> secondaryChildren = this.engine.getQueryEngine().findChildren(secondary);
+ Collection<Vertex> primaryChildren = this.engine.getQueryEngine().findChildren(primary);
+
+ mergeChildren(primary, secondary, primaryChildren, secondaryChildren, forceCopy);
+
+ Collection<Vertex> secondaryCousins = this.engine.getQueryEngine().findCousinVertices(secondary);
+ Collection<Vertex> primaryCousins = this.engine.getQueryEngine().findCousinVertices(primary);
+
+ secondaryCousins.removeAll(primaryCousins);
+ secondary.remove();
+ for (Vertex v : secondaryCousins) {
+ this.edgeSerializer.addEdgeIfPossible(g, v, primary);
+ }
+ }
+
+ private String getURI(Vertex v) throws UnsupportedEncodingException, AAIException {
+ Introspector obj = loader.introspectorFromName(v.<String>property(AAIProperties.NODE_TYPE).orElse(""));
+ this.serializer.dbToObject(Collections.singletonList(v), obj, 0, true, "false");
+ return obj.getURI();
+
+ }
+ private void mergeChildren(Vertex primary, Vertex secondary, Collection<Vertex> primaryChildren, Collection<Vertex> secondaryChildren, Map<String, Set<String>> forceCopy) throws UnsupportedEncodingException, AAIException {
+ Map<String, Vertex> primaryMap = uriMap(primaryChildren);
+ Map<String, Vertex> secondaryMap = uriMap(secondaryChildren);
+ Set<String> primaryKeys = new HashSet<>(primaryMap.keySet());
+ Set<String> secondaryKeys = new HashSet<>(secondaryMap.keySet());
+ primaryKeys.retainAll(secondaryKeys);
+ final Set<String> mergeItems = new HashSet<>(primaryKeys);
+ primaryKeys = new HashSet<>(primaryMap.keySet());
+ secondaryKeys = new HashSet<>(secondaryMap.keySet());
+ secondaryKeys.removeAll(primaryKeys);
+ final Set<String> copyItems = new HashSet<>(secondaryKeys);
+
+ for (String key : mergeItems) {
+ this.performMergeHelper(primaryMap.get(key), secondaryMap.get(key), forceCopy);
+ }
+
+ for (String key : copyItems) {
+ this.edgeSerializer.addTreeEdgeIfPossible(g, secondaryMap.get(key), primary);
+ this.serializer.getEdgeBetween(EdgeType.TREE, secondary, secondaryMap.get(key)).remove();
+ }
+
+ }
+
+ private Map<String, Vertex> uriMap(Collection<Vertex> vertices) throws UnsupportedEncodingException, AAIException {
+ final Map<String, Vertex> result = new HashMap<>();
+ for (Vertex v : vertices) {
+ result.put(getURI(v), v);
+ }
+ return result;
+ }
+
+ private void mergeProperties(Vertex primary, Vertex secondary, Map<String, Set<String>> forceCopy) throws AAIUnknownObjectException {
+ final String primaryType = primary.<String>property(AAIProperties.NODE_TYPE).orElse("");
+ final String secondaryType = secondary.<String>property(AAIProperties.NODE_TYPE).orElse("");
+
+ final Introspector secondaryObj = loader.introspectorFromName(secondaryType);
+ secondary.properties().forEachRemaining(prop -> {
+ if (!primary.property(prop.key()).isPresent() || forceCopy.getOrDefault(primaryType, new HashSet<String>()).contains(prop.key())) {
+ primary.property(prop.key(), prop.value());
+ }
+ if (primary.property(prop.key()).isPresent() && secondary.property(prop.key()).isPresent() && secondaryObj.isListType(prop.key())) {
+ mergeCollection(primary, prop.key(), secondary.values(prop.key()));
+ }
+ });
+ }
+ private void mergeCollection(Vertex primary, String propName, Iterator<Object> secondaryValues) {
+ secondaryValues.forEachRemaining(item -> {
+ primary.property(propName, item);
+ });
+ }
+
+
+ public static class Builder {
+ private final TransactionalGraphEngine engine;
+
+ private final DBSerializer serializer;
+ private EdgeSerializer edgeSerializer;
+
+ private final Loader loader;
+ private NotificationHelper helper = null;
+ private boolean hasNotifications = false;
+ public Builder(Loader loader, TransactionalGraphEngine engine, DBSerializer serializer) {
+ this.loader = loader;
+ this.engine = engine;
+ this.serializer = serializer;
+ }
+
+ public Builder addNotifications(NotificationHelper helper) {
+ this.helper = helper;
+ this.hasNotifications = true;
+ return this;
+ }
+
+ public Builder edgeSerializer(EdgeSerializer edgeSerializer){
+ this.edgeSerializer = edgeSerializer;
+ return this;
+ }
+
+ public EdgeSerializer getEdgeSerializer(){
+ return edgeSerializer;
+ }
+
+ public VertexMerge build() {
+ return new VertexMerge(this);
+ }
+
+ protected TransactionalGraphEngine getEngine() {
+ return engine;
+ }
+
+ protected DBSerializer getSerializer() {
+ return serializer;
+ }
+
+ protected Loader getLoader() {
+ return loader;
+ }
+
+ protected NotificationHelper getHelper() {
+ return helper;
+ }
+
+ protected boolean isHasNotifications() {
+ return hasNotifications;
+ }
+
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/ContainmentDeleteOtherVPropertyMigration.java b/src/main/java/org/onap/aai/migration/v12/ContainmentDeleteOtherVPropertyMigration.java
new file mode 100644
index 0000000..361e8bc
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/ContainmentDeleteOtherVPropertyMigration.java
@@ -0,0 +1,106 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+
+import java.util.Optional;
+
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.enums.EdgeProperty;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Migrator;
+import org.onap.aai.migration.Status;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+
+//@Enabled
+@MigrationPriority(-100)
+@MigrationDangerRating(10)
+public class ContainmentDeleteOtherVPropertyMigration extends Migrator {
+
+ private boolean success = true;
+
+ public ContainmentDeleteOtherVPropertyMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ //just for testing using test edge rule files
+ public ContainmentDeleteOtherVPropertyMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions, String edgeRulesFile) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ @Override
+ public void run() {
+ try {
+ engine.asAdmin().getTraversalSource().E().sideEffect(t -> {
+ Edge e = t.get();
+ logger.info("out vertex: " + e.outVertex().property("aai-node-type").value() +
+ " in vertex: " + e.inVertex().property("aai-node-type").value() +
+ " label : " + e.label());
+ if (e.property(EdgeProperty.CONTAINS.toString()).isPresent() &&
+ e.property(EdgeProperty.DELETE_OTHER_V.toString()).isPresent()) {
+ //in case of orphans
+ if (!("constrained-element-set".equals(e.inVertex().property("aai-node-type").value())
+ && "model-element".equals(e.outVertex().property("aai-node-type").value()))) {
+ //skip the weird horrible problem child edge
+ String containment = (String) e.property(EdgeProperty.CONTAINS.toString()).value();
+ if (AAIDirection.OUT.toString().equalsIgnoreCase(containment) ||
+ AAIDirection.IN.toString().equalsIgnoreCase(containment) ||
+ AAIDirection.BOTH.toString().equalsIgnoreCase(containment)) {
+ logger.info("updating delete-other-v property");
+ e.property(EdgeProperty.DELETE_OTHER_V.toString(), containment);
+ }
+ }
+ }
+ }).iterate();
+ } catch (Exception e) {
+ logger.info("error encountered " + e.getClass() + " " + e.getMessage() + " " + ExceptionUtils.getFullStackTrace(e));
+ logger.error("error encountered " + e.getClass() + " " + e.getMessage() + " " + ExceptionUtils.getFullStackTrace(e));
+ success = false;
+ }
+
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.empty();
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "migrate-containment-delete-other-v";
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/DeletePInterface.java b/src/main/java/org/onap/aai/migration/v12/DeletePInterface.java
new file mode 100644
index 0000000..1089b2f
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/DeletePInterface.java
@@ -0,0 +1,131 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Migrator;
+import org.onap.aai.migration.Status;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+@MigrationPriority(0)
+@MigrationDangerRating(0)
+public class DeletePInterface extends Migrator {
+ private boolean success = true;
+ private final GraphTraversalSource g;
+ public DeletePInterface(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.g = this.engine.asAdmin().getTraversalSource();
+ }
+
+ @Override
+ public void run() {
+ int count = 0;
+ int skipCount = 0;
+ int errorCount = 0;
+ logger.info("---------- Start deleting p-interfaces ----------");
+ List<Vertex> pIntfList;
+ try {
+ pIntfList = g.V().has(AAIProperties.NODE_TYPE, "p-interface").has("source-of-truth", "AAI-CSVP-INSTARAMS")
+ .where(this.engine.getQueryBuilder().createEdgeTraversal(EdgeType.TREE, "p-interface", "pnf")
+ .<GraphTraversal<?, ?>>getQuery()).toList();
+
+ if (pIntfList != null && !pIntfList.isEmpty()) {
+ for (Vertex pInterfV : pIntfList) {
+ try {
+ Collection<Vertex> cousins = this.engine.getQueryEngine().findCousinVertices(pInterfV);
+
+ Collection<Vertex> children = this.engine.getQueryEngine().findChildren(pInterfV);
+ if (cousins == null || cousins.isEmpty()) {
+ if (children == null || children.isEmpty()) {
+ logger.info("Delete p-interface: " + getVertexURI(pInterfV));
+ pInterfV.remove();
+ count++;
+ } else {
+ skipCount++;
+ logger.info("skip p-interface " + getVertexURI(pInterfV) + " due to an existing relationship");
+ }
+ } else {
+ skipCount++;
+ logger.info("skip p-interface " + getVertexURI(pInterfV) + " due to an existing relationship");
+ }
+ } catch (Exception e) {
+ success = false;
+ errorCount++;
+ logger.error("error occured in deleting p-interface " + getVertexURI(pInterfV) + ", "+ e);
+ }
+ }
+ logger.info ("\n \n ******* Final Summary for deleting p-interfaces Migration ********* \n");
+ logger.info("Number of p-interfaces removed: "+ count +"\n");
+ logger.info("Number of p-interfaces skipped: "+ skipCount +"\n");
+ logger.info("Number of p-interfaces failed to delete due to error : "+ errorCount +"\n");
+ }
+ } catch (AAIException e) {
+ success = false;
+ logger.error("error occured in deleting p-interfaces " + e);
+ }
+ }
+
+ private String getVertexURI(Vertex v) {
+ if (v != null) {
+ if (v.property("aai-uri").isPresent()) {
+ return v.property("aai-uri").value().toString();
+ } else {
+ return "Vertex ID: " + v.id().toString();
+ }
+ } else {
+ return "";
+ }
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[] { "p-interface" });
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "DeletePInterface";
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/EdgeReportForToscaMigration.java b/src/main/java/org/onap/aai/migration/v12/EdgeReportForToscaMigration.java
new file mode 100644
index 0000000..1bdddf3
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/EdgeReportForToscaMigration.java
@@ -0,0 +1,162 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+/*-
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.*;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+import java.util.*;
+
+@MigrationPriority(0)
+@MigrationDangerRating(0)
+public class EdgeReportForToscaMigration extends Migrator {
+
+ private boolean success = true;
+
+ public EdgeReportForToscaMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions){
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public void run() {
+ Vertex out = null;
+ Vertex in = null;
+ String label = "";
+ String outURI = "";
+ String inURI = "";
+ String parentCousinIndicator = "NONE";
+ String oldEdgeString = null;
+ List<String> edgeMissingParentProperty = new ArrayList<>();
+ StringBuilder sb = new StringBuilder();
+ Set<String> noURI = new HashSet<>();
+ sb.append("----------EDGES----------\n");
+
+ GraphTraversalSource g = engine.asAdmin().getTraversalSource();
+
+ try {
+ Set<Edge> edges = g.E().toSet();
+ for (Edge edge : edges) {
+ out = edge.outVertex();
+ in = edge.inVertex();
+ label = edge.label();
+ outURI = this.getVertexURI(out);
+ inURI = this.getVertexURI(in);
+ parentCousinIndicator = "NONE";
+ oldEdgeString = this.toStringForPrinting(edge, 1);
+
+ if (!outURI.startsWith("/")) {
+ noURI.add(outURI);
+ }
+ if (!inURI.startsWith("/")) {
+ noURI.add(inURI);
+ }
+
+ if (out == null || in == null) {
+ logger.error(edge.id() + " invalid because one vertex was null: out=" + edge.outVertex() + " in=" + edge.inVertex());
+ } else {
+
+ if (edge.property("contains-other-v").isPresent()) {
+ parentCousinIndicator = edge.property("contains-other-v").value().toString();
+ } else if (edge.property("isParent").isPresent()) {
+ if ((Boolean)edge.property("isParent").value()) {
+ parentCousinIndicator = "OUT";
+ } else if (edge.property("isParent-REV").isPresent() && (Boolean)edge.property("isParent-REV").value()) {
+ parentCousinIndicator = "IN";
+ }
+ } else {
+ edgeMissingParentProperty.add(this.toStringForPrinting(edge, 1));
+ }
+
+ sb.append(outURI + "|" + label + "|" + inURI + "|" + parentCousinIndicator + "\n");
+ }
+ }
+ } catch(Exception ex){
+ logger.error("exception occurred during migration, failing: out=" + out + " in=" + in + "edge=" + oldEdgeString, ex);
+ success = false;
+ }
+ sb.append("--------EDGES END--------\n");
+
+ logger.info(sb.toString());
+ edgeMissingParentProperty.forEach(s -> logger.warn("Edge Missing Parent Property: " + s));
+ logger.info("Edge Missing Parent Property Count: " + edgeMissingParentProperty.size());
+ logger.info("Vertex Missing URI Property Count: " + noURI.size());
+
+ }
+
+ private String getVertexURI(Vertex v) {
+ if (v.property("aai-uri").isPresent()) {
+ return v.property("aai-uri").value().toString();
+ } else {
+ return v.id().toString() + "(" + v.property("aai-node-type").value().toString() + ")";
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.empty();
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "edge-report-for-tosca-migration";
+ }
+
+ @Override
+ public void commit() {
+ engine.rollback();
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateModelVerDistriubutionStatusProperty.java b/src/main/java/org/onap/aai/migration/v12/MigrateModelVerDistriubutionStatusProperty.java
new file mode 100644
index 0000000..c09643f
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/MigrateModelVerDistriubutionStatusProperty.java
@@ -0,0 +1,85 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.*;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+import java.util.Optional;
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+public class MigrateModelVerDistriubutionStatusProperty extends Migrator{
+
+ private final String PARENT_NODE_TYPE = "model-ver";
+ private boolean success = true;
+
+ public MigrateModelVerDistriubutionStatusProperty(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+
+
+ @Override
+ public void run() {
+
+
+ GraphTraversal<Vertex, Vertex> f = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE,"model-ver");
+
+ while(f.hasNext()) {
+ Vertex v = f.next();
+ try {
+ v.property("distribution-status", "DISTRIBUTION_COMPLETE_OK");
+ logger.info("changed model-ver.distribution-status property value for model-version-id: " + v.property("model-version-id").value());
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ success = false;
+ logger.error("encountered exception for model-version-id:" + v.property("model-version-id").value(), e);
+ }
+ }
+ }
+
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{PARENT_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateModelVerDistriubutionStatusProperty";
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/MigrateServiceInstanceToConfiguration.java b/src/main/java/org/onap/aai/migration/v12/MigrateServiceInstanceToConfiguration.java
new file mode 100644
index 0000000..b4208af
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/MigrateServiceInstanceToConfiguration.java
@@ -0,0 +1,193 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+
+import java.io.UnsupportedEncodingException;
+import java.util.Iterator;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.Introspector;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Migrator;
+import org.onap.aai.migration.Status;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+//@Enabled
+@MigrationPriority(10)
+@MigrationDangerRating(10)
+public class MigrateServiceInstanceToConfiguration extends Migrator {
+
+ private boolean success = true;
+ private final String CONFIGURATION_NODE_TYPE = "configuration";
+ private final String SERVICE_INSTANCE_NODE_TYPE = "service-instance";
+ private Introspector configObj;
+
+ public MigrateServiceInstanceToConfiguration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ try {
+ this.configObj = this.loader.introspectorFromName(CONFIGURATION_NODE_TYPE);
+ } catch (AAIUnknownObjectException e) {
+ this.configObj = null;
+ }
+ }
+
+ @Override
+ public void run() {
+ Vertex serviceInstance = null;
+ Vertex configuration = null;
+ String serviceInstanceId = "", tunnelBandwidth = "";
+ String bandwidthTotal, configType, nodeType;
+ GraphTraversal<Vertex, Vertex> serviceInstanceItr;
+ Iterator<Vertex> configurationItr;
+
+ try {
+ serviceInstanceItr = this.engine.asAdmin().getTraversalSource().V()
+ .has(AAIProperties.NODE_TYPE, P.within(getAffectedNodeTypes().get()))
+ .where(this.engine.getQueryBuilder()
+ .createEdgeTraversal(EdgeType.TREE, "service-instance", "service-subscription")
+ .getVerticesByProperty("service-type", "DHV")
+ .<GraphTraversal<?, ?>>getQuery());
+
+ if (serviceInstanceItr == null || !serviceInstanceItr.hasNext()) {
+ logger.info("No servince-instance nodes found with service-type of DHV");
+ return;
+ }
+
+ // iterate through all service instances of service-type DHV
+ while (serviceInstanceItr.hasNext()) {
+ serviceInstance = serviceInstanceItr.next();
+
+ if (serviceInstance != null && serviceInstance.property("bandwidth-total").isPresent()) {
+ serviceInstanceId = serviceInstance.value("service-instance-id");
+ logger.info("Processing service instance with id=" + serviceInstanceId);
+ bandwidthTotal = serviceInstance.value("bandwidth-total");
+
+ if (bandwidthTotal != null && !bandwidthTotal.isEmpty()) {
+
+ // check for existing edges to configuration nodes
+ configurationItr = serviceInstance.vertices(Direction.OUT, "has");
+
+ // create new configuration node if service-instance does not have existing ones
+ if (!configurationItr.hasNext()) {
+ logger.info(serviceInstanceId + " has no existing configuration nodes, creating new node");
+ createConfigurationNode(serviceInstance, bandwidthTotal);
+ continue;
+ }
+
+ // in case if configuration nodes exist, but none are DHV
+ boolean hasDHVConfig = false;
+
+ // service-instance has existing configuration nodes
+ while (configurationItr.hasNext()) {
+ configuration = configurationItr.next();
+ nodeType = configuration.value("aai-node-type").toString();
+
+ if (configuration != null && "configuration".equalsIgnoreCase(nodeType)) {
+ logger.info("Processing configuration node with id=" + configuration.property("configuration-id").value());
+ configType = configuration.value("configuration-type");
+ logger.info("Configuration type: " + configType);
+
+ // if configuration-type is DHV, update tunnel-bandwidth to bandwidth-total value
+ if ("DHV".equalsIgnoreCase(configType)) {
+ if (configuration.property("tunnel-bandwidth").isPresent()) {
+ tunnelBandwidth = configuration.value("tunnel-bandwidth");
+ } else {
+ tunnelBandwidth = "";
+ }
+
+ logger.info("Existing tunnel-bandwidth: " + tunnelBandwidth);
+ configuration.property("tunnel-bandwidth", bandwidthTotal);
+ touchVertexProperties(configuration, false);
+ logger.info("Updated tunnel-bandwidth: " + configuration.value("tunnel-bandwidth"));
+ hasDHVConfig = true;
+ }
+ }
+ }
+
+ // create new configuration node if none of existing config nodes are of type DHV
+ if (!hasDHVConfig) {
+ logger.info(serviceInstanceId + " has existing configuration nodes, but none are DHV, create new node");
+ createConfigurationNode(serviceInstance, bandwidthTotal);
+ }
+ }
+ }
+ }
+ } catch (AAIException | UnsupportedEncodingException e) {
+ logger.error("Caught exception while processing service instance with id=" + serviceInstanceId + " | " + e.toString());
+ success = false;
+ }
+ }
+
+ private void createConfigurationNode(Vertex serviceInstance, String bandwidthTotal) throws UnsupportedEncodingException, AAIException {
+ // create new vertex
+ Vertex configurationNode = serializer.createNewVertex(configObj);
+
+ // configuration-id: UUID format
+ String configurationUUID = UUID.randomUUID().toString();
+ configObj.setValue("configuration-id", configurationUUID);
+
+ // configuration-type: DHV
+ configObj.setValue("configuration-type", "DHV");
+
+ // migrate the bandwidth-total property from the service-instance to the
+ // tunnel-bandwidth property of the related configuration object
+ configObj.setValue("tunnel-bandwidth", bandwidthTotal);
+
+ // create edge between service instance and configuration: cousinEdge(out, in)
+ createCousinEdge(serviceInstance, configurationNode);
+
+ // serialize edge & vertex, takes care of everything
+ serializer.serializeSingleVertex(configurationNode, configObj, "migrations");
+ logger.info("Created configuration node with uuid=" + configurationUUID + ", tunnel-bandwidth=" + bandwidthTotal);
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[] {SERVICE_INSTANCE_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "service-instance-to-configuration";
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/SDWANSpeedChangeMigration.java b/src/main/java/org/onap/aai/migration/v12/SDWANSpeedChangeMigration.java
new file mode 100644
index 0000000..b420c57
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/SDWANSpeedChangeMigration.java
@@ -0,0 +1,258 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.*;
+import org.onap.aai.edges.enums.EdgeType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+import java.util.*;
+
+@MigrationPriority(1)
+@MigrationDangerRating(1)
+//@Enabled
+public class SDWANSpeedChangeMigration extends Migrator {
+
+ private final String PARENT_NODE_TYPE = "alloted-resource";
+ private boolean success = true;
+
+ Vertex allottedRsrcVertex;
+
+ Map<String, String> bandwidthMap = new HashMap<>();
+ Set<String> bandWidthSet = new HashSet<>();
+
+ GraphTraversal<Vertex, Vertex> allottedRsrcTraversal;
+ GraphTraversal<Vertex, Vertex> tunnelXConnectTraversal;
+ GraphTraversal<Vertex, Vertex> pinterfaceTraversal;
+ GraphTraversal<Vertex, Vertex> plinkTraversal;
+
+ public SDWANSpeedChangeMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ bandWidthSet.add("bandwidth-up-wan1");
+ bandWidthSet.add("bandwidth-down-wan1");
+ bandWidthSet.add("bandwidth-up-wan2");
+ bandWidthSet.add("bandwidth-down-wan2");
+ }
+
+
+ @Override
+ public void run() {
+
+ logger.info("Started the migration "+ getMigrationName());
+
+ try {
+
+ allottedRsrcTraversal = this.engine.asAdmin().getTraversalSource().V()
+ .has("aai-node-type", "service-subscription")
+ .has("service-type", "DHV")
+ .in("org.onap.relationships.inventory.BelongsTo")
+ .has("aai-node-type", "service-instance")
+ .out("org.onap.relationships.inventory.Uses")
+ .has("aai-node-type", "allotted-resource")
+ .where(
+ this.engine.getQueryBuilder()
+ .createEdgeTraversal(EdgeType.TREE, "allotted-resource", "service-instance")
+ .createEdgeTraversal(EdgeType.TREE, "service-instance", "service-subscription")
+ .<GraphTraversal<Vertex, Vertex>>getQuery()
+ .has("service-type", "VVIG")
+ );
+
+ if(!(allottedRsrcTraversal.hasNext())){
+
+ logger.info("unable to find allotted resource to DHV as cousin and child of VVIG");
+ }
+
+ while (allottedRsrcTraversal.hasNext()) {
+ bandwidthMap.clear();
+
+ allottedRsrcVertex = allottedRsrcTraversal.next();
+ String allottedResourceId = allottedRsrcVertex.property("id").value().toString();
+ logger.info("Found an allotted resource with id " + allottedResourceId);
+
+ tunnelXConnectTraversal = this.engine.asAdmin().getTraversalSource()
+ .V(allottedRsrcVertex)
+ .in("org.onap.relationships.inventory.BelongsTo")
+ .has("aai-node-type", "tunnel-xconnect");
+
+ if (tunnelXConnectTraversal != null && tunnelXConnectTraversal.hasNext()) {
+ Vertex xConnect = tunnelXConnectTraversal.next();
+ String tunnelId = xConnect.property("id").value().toString();
+ logger.info("Found an tunnelxconnect object with id " + tunnelId);
+ extractBandwidthProps(xConnect);
+ modifyPlink(allottedRsrcVertex);
+ } else {
+ logger.info("Unable to find the tunnel connect for the current allotted resource traversal");
+ }
+
+ }
+ } catch (AAIException e) {
+ e.printStackTrace();
+ success = false;
+ }
+
+ logger.info("Successfully finished the " + getMigrationName());
+ }
+
+ public void extractBandwidthProps(Vertex vertex) {
+ logger.info("Trying to extract bandwith props");
+ bandWidthSet.stream().forEach((key) -> {
+ if (vertex.property(key).isPresent()) {
+ bandwidthMap.put(key, vertex.property(key).value().toString());
+ }
+ });
+ logger.info("Extracted bandwith props for tunnelXConnect " +vertex.value("id"));
+ }
+
+ public void modifyPlink(Vertex v) {
+
+ try {
+ pinterfaceTraversal = this.engine.asAdmin().getTraversalSource().V(v)
+ .in("org.onap.relationships.inventory.Uses").has("aai-node-type", "service-instance")
+ .where(
+ __.out("org.onap.relationships.inventory.BelongsTo")
+ .has("aai-node-type", "service-subscription")
+ .has("service-type", "DHV")
+ )
+ .out("org.onap.relationships.inventory.ComposedOf").has("aai-node-type", "generic-vnf")
+ .out("tosca.relationships.HostedOn").has("aai-node-type", "vserver")
+ .out("tosca.relationships.HostedOn").has("aai-node-type", "pserver")
+ .in("tosca.relationships.network.BindsTo").has("aai-node-type", "p-interface");
+ } catch (Exception e) {
+ logger.info("error trying to find p interfaces");
+ }
+
+
+ while (pinterfaceTraversal.hasNext()) {
+
+ Vertex pInterfaceVertex = pinterfaceTraversal.next();
+
+ String pinterfaceName = pInterfaceVertex.property("interface-name").value().toString();
+ logger.info("p-interface "+ pinterfaceName + " found from traversal from allotted-resource " +v.value("id"));
+ String[] parts = pinterfaceName.split("/");
+
+ if (parts[parts.length - 1].equals("10")) {
+
+ logger.info("Found the pinterface with the interface name ending with /10");
+
+ try {
+ plinkTraversal = this.engine.asAdmin().getTraversalSource()
+ .V(pInterfaceVertex)
+ .out("tosca.relationships.network.LinksTo")
+ .has("aai-node-type", "physical-link");
+ } catch (Exception e) {
+ logger.info("error trying to find the p Link for /10");
+ }
+ if (plinkTraversal != null && plinkTraversal.hasNext()) {
+ Vertex pLink = plinkTraversal.next();
+
+
+ if ( bandwidthMap.containsKey("bandwidth-up-wan1")
+ && bandwidthMap.containsKey("bandwidth-down-wan1")
+ && !(("").equals(bandwidthMap.get("bandwidth-up-wan1").replaceAll("[^0-9]", "").trim()))
+ && !(("").equals(bandwidthMap.get("bandwidth-down-wan1").replaceAll("[^0-9]", "").trim())))
+ {
+
+ pLink.property("service-provider-bandwidth-up-value", Integer.valueOf(bandwidthMap.get("bandwidth-up-wan1").replaceAll("[^0-9]", "").trim()));
+ pLink.property("service-provider-bandwidth-up-units", "Mbps");
+ pLink.property("service-provider-bandwidth-down-value", Integer.valueOf(bandwidthMap.get("bandwidth-down-wan1").replaceAll("[^0-9]", "").trim()));
+ pLink.property("service-provider-bandwidth-down-units", "Mbps");
+ logger.info("Successfully modified the plink with link name ", pLink.property("link-name").value().toString());
+ this.touchVertexProperties(pLink, false);
+ } else {
+ logger.info("missing up and down vals for the plink with link name ", pLink.property("link-name").value().toString());
+ }
+
+
+ } else {
+ logger.info("missing plink for p interface" + pinterfaceName);
+ }
+
+ }
+
+ if (parts[parts.length - 1].equals("11")) {
+
+ logger.info("Found the pinterface with the interface name ending with /11");
+ try {
+ plinkTraversal = this.engine.asAdmin()
+ .getTraversalSource()
+ .V(pInterfaceVertex)
+ .out("tosca.relationships.network.LinksTo")
+ .has("aai-node-type", "physical-link");
+ } catch (Exception e) {
+ logger.info("error trying to find the p Link for /11");
+ }
+
+ if (plinkTraversal != null && plinkTraversal.hasNext()) {
+ Vertex pLink = plinkTraversal.next();
+
+
+ if ( bandwidthMap.containsKey("bandwidth-up-wan2")
+ && bandwidthMap.containsKey("bandwidth-down-wan2")
+ && !(("").equals(bandwidthMap.get("bandwidth-up-wan2").replaceAll("[^0-9]", "").trim()))
+ && !(("").equals(bandwidthMap.get("bandwidth-down-wan2").replaceAll("[^0-9]", "").trim())))
+ {
+ pLink.property("service-provider-bandwidth-up-value", Integer.valueOf(bandwidthMap.get("bandwidth-up-wan2").replaceAll("[^0-9]", "").trim()));
+ pLink.property("service-provider-bandwidth-up-units", "Mbps");
+ pLink.property("service-provider-bandwidth-down-value", Integer.valueOf(bandwidthMap.get("bandwidth-down-wan2").replaceAll("[^0-9]", "").trim()));
+ pLink.property("service-provider-bandwidth-down-units", "Mbps");
+ logger.info("Successfully modified the plink with link name ", pLink.property("link-name").value().toString());
+ this.touchVertexProperties(pLink, false);
+ } else {
+ logger.error("missing up and down vals for the plink with link name ", pLink.property("link-name").value().toString());
+ }
+
+ } else {
+ logger.info("missing plink for p interface" + pinterfaceName);
+ }
+ }
+ }
+ }
+
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+
+ return Optional.of(new String[]{PARENT_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "SDWANSpeedChangeMigration";
+ }
+
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/UpdateAaiUriIndexMigration.java b/src/main/java/org/onap/aai/migration/v12/UpdateAaiUriIndexMigration.java
new file mode 100644
index 0000000..33689b5
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/UpdateAaiUriIndexMigration.java
@@ -0,0 +1,328 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+
+import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.PropertyKey;
+import org.janusgraph.core.schema.SchemaAction;
+import org.janusgraph.core.schema.SchemaStatus;
+import org.janusgraph.core.schema.JanusGraphIndex;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.graphdb.database.management.ManagementSystem;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.migration.*;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+
+import java.time.temporal.ChronoUnit;
+import java.util.*;
+
+/**
+ * Remove old aai-uri index per
+ * https://github.com/JanusGraph/janusgraph/wiki/Indexing
+ */
+
+@Enabled
+
+@MigrationPriority(500)
+@MigrationDangerRating(1000)
+public class UpdateAaiUriIndexMigration extends Migrator {
+
+ private final SchemaVersion version;
+ private final ModelType introspectorFactoryType;
+ private GraphTraversalSource g;
+ private JanusGraphManagement graphMgmt;
+ private Status status = Status.SUCCESS;
+
+ private String retiredName = AAIProperties.AAI_URI + "-RETIRED-" + System.currentTimeMillis();
+
+ /**
+ * Instantiates a new migrator.
+ *
+ * @param engine
+ */
+ public UpdateAaiUriIndexMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) throws AAIException {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ version = schemaVersions.getDefaultVersion();
+ introspectorFactoryType = ModelType.MOXY;
+ loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ g = this.engine.asAdmin().getTraversalSource();
+ this.engine.rollback();
+ graphMgmt = engine.asAdmin().getManagementSystem();
+
+ }
+
+ @Override
+ public Status getStatus() {
+ return status;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.empty();
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "UpdateAaiUriIndex";
+ }
+
+ @Override
+ public void run() {
+
+ // close all but current open titan instances
+ closeAllButCurrentInstances();
+
+ // get all indexes containing aai-uri
+ Set<IndexDetails> indexes = getIndexesWithAaiUri();
+ logger.info("Found " + indexes.size() + " aai uri index.");
+ indexes.stream().map(s -> "\t" + s.getIndexName() + " : " + s.getPropertyName() + " : " + s.getStatus() ).forEach(System.out::println);
+
+ renameAaiUriIndex(indexes);
+
+ // remove all of the aai-uri indexes that are in the list
+ removeIndexes(indexes);
+
+ //retire old property
+ verifyGraphManagementIsOpen();
+ PropertyKey aaiUri = graphMgmt.getPropertyKey(AAIProperties.AAI_URI);
+ if (aaiUri != null) {
+ graphMgmt.changeName(aaiUri, retiredName);
+ }
+ graphMgmt.commit();
+
+ //remove all aai uri keys
+ logger.info("Remove old keys.");
+ dropAllKeyProperties(indexes);
+
+ // add aai-uri unique index
+ logger.info("Create new unique aai-uri index");
+ createUniqueAaiUriIndex();
+
+
+ // change index status to ENABLED STATE
+ logger.info("Enable index");
+ enableIndex();
+
+ this.engine.startTransaction();
+
+ logger.info("Checking and dropping retired properties.");
+ g = this.engine.asAdmin().getTraversalSource();
+ g.V().has(retiredName).properties(retiredName).drop().iterate();
+ logger.info("Done.");
+ }
+
+
+ protected void createUniqueAaiUriIndex() {
+ verifyGraphManagementIsOpen();
+ // create new aaiuri property
+ PropertyKey aaiUriProperty = graphMgmt.getPropertyKey(AAIProperties.AAI_URI);
+ if (aaiUriProperty == null) {
+ logger.info("Creating new aai-uri property.");
+ aaiUriProperty = graphMgmt.makePropertyKey(AAIProperties.AAI_URI).dataType(String.class)
+ .cardinality(Cardinality.SINGLE).make();
+ }
+ logger.info("Creating new aai-uri index.");
+ graphMgmt.buildIndex(AAIProperties.AAI_URI, Vertex.class).addKey(aaiUriProperty).unique().buildCompositeIndex();
+ graphMgmt.commit();
+ }
+
+ private void dropAllKeyProperties(Set<IndexDetails> indexes) {
+ indexes.stream().map(e -> e.getPropertyName()).distinct().forEach(p -> {
+ verifyGraphManagementIsOpen();
+ if (graphMgmt.getPropertyKey(p) != null) {
+ graphMgmt.getPropertyKey(p).remove();
+ }
+ graphMgmt.commit();
+ });
+ }
+
+ private void renameAaiUriIndex(Set<IndexDetails> indexes) {
+ verifyGraphManagementIsOpen();
+ indexes.stream().filter(s -> s.getIndexName().equals(AAIProperties.AAI_URI)).forEach( s -> {
+ JanusGraphIndex index = graphMgmt.getGraphIndex(s.getIndexName());
+ graphMgmt.changeName(index, retiredName);
+ s.setIndexName(retiredName);
+ });
+ graphMgmt.commit();
+ }
+
+ private void removeIndexes(Set<IndexDetails> indexes) {
+
+ for (IndexDetails index : indexes) {
+ verifyGraphManagementIsOpen();
+
+ JanusGraphIndex aaiUriIndex = graphMgmt.getGraphIndex(index.getIndexName());
+
+ if (!index.getStatus().equals(SchemaStatus.DISABLED)) {
+ logger.info("Disabling index: " + index.getIndexName());
+ logger.info("\tCurrent state: " + aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(index.getPropertyName())));
+
+ graphMgmt.updateIndex(aaiUriIndex, SchemaAction.DISABLE_INDEX);
+ graphMgmt.commit();
+ try {
+ ManagementSystem.awaitGraphIndexStatus(AAIGraph.getInstance().getGraph(), index.getIndexName())
+ .timeout(10, ChronoUnit.MINUTES)
+ .status(SchemaStatus.DISABLED)
+ .call();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ verifyGraphManagementIsOpen();
+ aaiUriIndex = graphMgmt.getGraphIndex(index.getIndexName());
+ if (aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(index.getPropertyName())).equals(SchemaStatus.DISABLED)) {
+ logger.info("Removing index: " + index.getIndexName());
+ graphMgmt.updateIndex(aaiUriIndex, SchemaAction.REMOVE_INDEX);
+ graphMgmt.commit();
+ }
+ if(graphMgmt.isOpen()) {
+ graphMgmt.commit();
+ }
+ }
+
+ }
+
+ protected Set<IndexDetails> getIndexesWithAaiUri() {
+ verifyGraphManagementIsOpen();
+ Set<IndexDetails> aaiUriIndexName = new HashSet<>();
+
+ Iterator<JanusGraphIndex> titanIndexes = graphMgmt.getGraphIndexes(Vertex.class).iterator();
+ JanusGraphIndex titanIndex;
+ while (titanIndexes.hasNext()) {
+ titanIndex = titanIndexes.next();
+ if (titanIndex.name().contains(AAIProperties.AAI_URI) && titanIndex.getFieldKeys().length > 0) {
+ logger.info("Found aai-uri index: " + titanIndex.name());
+ aaiUriIndexName.add(new IndexDetails(titanIndex.name(), titanIndex.getIndexStatus(titanIndex.getFieldKeys()[0]), titanIndex.getFieldKeys()[0].name()));
+ }
+ }
+ graphMgmt.rollback();
+ return aaiUriIndexName;
+ }
+
+ private void closeAllButCurrentInstances() {
+ verifyGraphManagementIsOpen();
+ logger.info("Closing all but current titan instances.");
+ graphMgmt.getOpenInstances().stream().filter(s -> !s.contains("(current)")).forEach(s -> {
+ logger.info("\t"+s);
+ graphMgmt.forceCloseInstance(s);
+ });
+ graphMgmt.commit();
+ }
+
+
+ private void verifyGraphManagementIsOpen() {
+ if (!graphMgmt.isOpen()) {
+ graphMgmt = this.engine.asAdmin().getManagementSystem();
+ }
+ }
+
+ private void enableIndex() {
+ verifyGraphManagementIsOpen();
+ JanusGraphIndex aaiUriIndex = graphMgmt.getGraphIndex(AAIProperties.AAI_URI);
+ SchemaStatus schemaStatus = aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(AAIProperties.AAI_URI));
+ if (schemaStatus.equals(SchemaStatus.INSTALLED)) {
+ logger.info("Registering index: " + AAIProperties.AAI_URI);
+ logger.info("\tCurrent state: " + schemaStatus);
+
+ graphMgmt.updateIndex(aaiUriIndex, SchemaAction.REGISTER_INDEX);
+ graphMgmt.commit();
+ try {
+ ManagementSystem.awaitGraphIndexStatus(AAIGraph.getInstance().getGraph(), AAIProperties.AAI_URI)
+ .timeout(10, ChronoUnit.MINUTES)
+ .status(SchemaStatus.REGISTERED)
+ .call();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ verifyGraphManagementIsOpen();
+ aaiUriIndex = graphMgmt.getGraphIndex(AAIProperties.AAI_URI);
+ schemaStatus = aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(AAIProperties.AAI_URI));
+ if (schemaStatus.equals(SchemaStatus.REGISTERED)) {
+ logger.info("Enabling index: " + AAIProperties.AAI_URI);
+ logger.info("\tCurrent state: " + schemaStatus);
+
+ graphMgmt.updateIndex(aaiUriIndex, SchemaAction.ENABLE_INDEX);
+ graphMgmt.commit();
+ try {
+ ManagementSystem.awaitGraphIndexStatus(AAIGraph.getInstance().getGraph(), AAIProperties.AAI_URI)
+ .timeout(10, ChronoUnit.MINUTES)
+ .status(SchemaStatus.ENABLED)
+ .call();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ verifyGraphManagementIsOpen();
+ aaiUriIndex = graphMgmt.getGraphIndex(AAIProperties.AAI_URI);
+ schemaStatus = aaiUriIndex.getIndexStatus(graphMgmt.getPropertyKey(AAIProperties.AAI_URI));
+ logger.info("Final state: " + schemaStatus);
+ graphMgmt.rollback();
+ }
+
+ private class IndexDetails {
+ private String indexName;
+ private SchemaStatus status;
+ private String propertyName;
+
+ public IndexDetails(String indexName, SchemaStatus status, String propertyName) {
+ this.indexName = indexName;
+ this.status = status;
+ this.propertyName = propertyName;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public SchemaStatus getStatus() {
+ return status;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+
+ public void setIndexName(String indexName) {
+ this.indexName = indexName;
+ }
+
+ public void setStatus(SchemaStatus status) {
+ this.status = status;
+ }
+
+ public void setPropertyName(String propertyName) {
+ this.propertyName = propertyName;
+ }
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/v12/UriMigration.java b/src/main/java/org/onap/aai/migration/v12/UriMigration.java
new file mode 100644
index 0000000..cb0926e
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v12/UriMigration.java
@@ -0,0 +1,180 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v12;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.enums.EdgeProperty;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
+import org.onap.aai.migration.*;
+import org.onap.aai.edges.enums.AAIDirection;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.springframework.web.util.UriUtils;
+
+import javax.ws.rs.core.UriBuilder;
+import java.io.UnsupportedEncodingException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Enabled
+
+@MigrationPriority(1000)
+@MigrationDangerRating(1000)
+public class UriMigration extends Migrator {
+
+ private final SchemaVersion version;
+ private final ModelType introspectorFactoryType;
+ private GraphTraversalSource g;
+
+ private Map<String, UriBuilder> nodeTypeToUri;
+ private Map<String, Set<String>> nodeTypeToKeys;
+
+ protected Set<Object> seen = new HashSet<>();
+
+ /**
+ * Instantiates a new migrator.
+ *
+ * @param engine
+ */
+ public UriMigration(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) throws AAIException {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ version = schemaVersions.getDefaultVersion();
+ introspectorFactoryType = ModelType.MOXY;
+ loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ g = this.engine.asAdmin().getTraversalSource();
+ this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.getMigrationName());
+
+ }
+
+ @Override
+ public void run() {
+ long start = System.currentTimeMillis();
+ nodeTypeToUri = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect(
+ Collectors.toMap(
+ e -> e.getKey(),
+ e -> UriBuilder.fromPath(e.getValue().getFullGenericURI().replaceAll("\\{"+ e.getKey() + "-", "{"))
+ ));
+
+ nodeTypeToKeys = loader.getAllObjects().entrySet().stream().filter(e -> e.getValue().getGenericURI().contains("{")).collect(
+ Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().getKeys()
+ ));
+
+ Set<String> topLevelNodeTypes = loader.getAllObjects().entrySet().stream()
+ .filter(e -> e.getValue().isTopLevel()).map(Map.Entry::getKey)
+ .collect(Collectors.toSet());
+
+ logger.info("Top level count : " + topLevelNodeTypes.size());
+ topLevelNodeTypes.stream().forEach(topLevelNodeType -> {
+ Set<Vertex> parentSet = g.V().has(AAIProperties.NODE_TYPE, topLevelNodeType).toSet();
+ logger.info(topLevelNodeType + " : " + parentSet.size());
+ try {
+ this.verifyOrAddUri("", parentSet);
+ } catch (AAIUnknownObjectException e) {
+ e.printStackTrace();
+ } catch (AAIException e) {
+ e.printStackTrace();
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
+ });
+ logger.info("RUNTIME: " + (System.currentTimeMillis() - start));
+ logger.info("NO URI: " + g.V().hasNot(AAIProperties.AAI_URI).count().next());
+ logger.info("NUM VERTEXES SEEN: " + seen.size());
+ seen = new HashSet<>();
+
+ }
+
+ protected void verifyOrAddUri(String parentUri, Set<Vertex> vertexSet) throws UnsupportedEncodingException, AAIException {
+ String correctUri;
+ for (Vertex v : vertexSet) {
+ seen.add(v.id());
+ //if there is an issue generating the uri catch, log and move on;
+ try {
+ correctUri = parentUri + this.getUriForVertex(v);
+ } catch (Exception e) {
+ logger.error("Vertex has issue generating uri " + e.getMessage() + "\n\t" + this.asString(v));
+ continue;
+ }
+ try {
+ v.property(AAIProperties.AAI_URI, correctUri);
+ } catch (Exception e) {
+ logger.info(e.getMessage() + "\n\t" + this.asString(v));
+ }
+ if (!v.property(AAIProperties.AAI_UUID).isPresent()) {
+ v.property(AAIProperties.AAI_UUID, UUID.randomUUID().toString());
+ }
+ this.verifyOrAddUri(correctUri, getChildren(v));
+ }
+ }
+
+ protected Set<Vertex> getChildren(Vertex v) {
+
+ Set<Vertex> children = g.V(v).bothE().not(__.has(EdgeProperty.CONTAINS.toString(), AAIDirection.NONE.toString())).otherV().toSet();
+
+ return children.stream().filter(child -> !seen.contains(child.id())).collect(Collectors.toSet());
+ }
+
+ protected String getUriForVertex(Vertex v) {
+ String aaiNodeType = v.property(AAIProperties.NODE_TYPE).value().toString();
+
+
+ Map<String, String> parameters = this.nodeTypeToKeys.get(aaiNodeType).stream().collect(Collectors.toMap(
+ key -> key,
+ key -> encodeProp(v.property(key).value().toString())
+ ));
+
+ return this.nodeTypeToUri.get(aaiNodeType).buildFromEncodedMap(parameters).toString();
+ }
+
+ private static String encodeProp(String s) {
+ try {
+ return UriUtils.encode(s, "UTF-8");
+ } catch (UnsupportedEncodingException e) {
+ return "";
+ }
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.empty();
+ }
+
+ @Override
+ public String getMigrationName() {
+ return UriMigration.class.getSimpleName();
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateBooleanDefaultsToFalse.java b/src/main/java/org/onap/aai/migration/v13/MigrateBooleanDefaultsToFalse.java
new file mode 100644
index 0000000..89a9459
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateBooleanDefaultsToFalse.java
@@ -0,0 +1,114 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(1)
+@MigrationDangerRating(1)
+public class MigrateBooleanDefaultsToFalse extends ValueMigrator {
+ protected static final String VNF_NODE_TYPE = "generic-vnf";
+ protected static final String VSERVER_NODE_TYPE = "vserver";
+ protected static final String VNFC_NODE_TYPE = "vnfc";
+ protected static final String L3NETWORK_NODE_TYPE = "l3-network";
+ protected static final String SUBNET_NODE_TYPE = "subnet";
+ protected static final String LINTERFACE_NODE_TYPE = "l-interface";
+ protected static final String VFMODULE_NODE_TYPE = "vf-module";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair1;
+ private static Map<String, Boolean> pair2;
+ private static Map<String, Boolean> pair3;
+ private static Map<String, Boolean> pair4;
+ private static Map<String, Boolean> pair5;
+ private static Map<String, Boolean> pair6;
+
+ public MigrateBooleanDefaultsToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setBooleanDefaultsToFalse(), false);
+
+ }
+
+ private static Map<String, Map> setBooleanDefaultsToFalse(){
+ map = new HashMap<>();
+ pair1 = new HashMap<>();
+ pair2 = new HashMap<>();
+ pair3 = new HashMap<>();
+ pair4 = new HashMap<>();
+ pair5 = new HashMap<>();
+ pair6 = new HashMap<>();
+
+
+ pair1.put("is-closed-loop-disabled", false);
+ map.put("generic-vnf", pair1);
+ map.put("vnfc", pair1);
+ map.put("vserver", pair1);
+
+ pair2.put("is-bound-to-vpn", false);
+ pair2.put("is-provider-network", false);
+ pair2.put("is-shared-network", false);
+ pair2.put("is-external-network", false);
+ map.put("l3-network", pair2);
+
+ pair3.put("dhcp-enabled", false);
+ map.put("subnet", pair3);
+
+ pair4.put("is-port-mirrored", false);
+ pair4.put("is-ip-unnumbered", false);
+ map.put("l-interface", pair4);
+
+ pair5.put("is-base-vf-module", false);
+ map.put("vf-module", pair5);
+
+ pair6.put("is-ip-unnumbered", false);
+ map.put("vlan", pair6);
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{VNF_NODE_TYPE,VSERVER_NODE_TYPE,VNFC_NODE_TYPE,L3NETWORK_NODE_TYPE,SUBNET_NODE_TYPE,LINTERFACE_NODE_TYPE,VFMODULE_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateBooleanDefaultsToFalse";
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInMaintDefaultToFalse.java b/src/main/java/org/onap/aai/migration/v13/MigrateInMaintDefaultToFalse.java
new file mode 100644
index 0000000..1773038
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateInMaintDefaultToFalse.java
@@ -0,0 +1,98 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Status;
+import org.onap.aai.migration.ValueMigrator;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(1)
+@MigrationDangerRating(1)
+public class MigrateInMaintDefaultToFalse extends ValueMigrator {
+
+ protected static final String VNF_NODE_TYPE = "generic-vnf";
+ protected static final String LINTERFACE_NODE_TYPE = "l-interface";
+ protected static final String LAG_INTERFACE_NODE_TYPE = "lag-interface";
+ protected static final String LOGICAL_LINK_NODE_TYPE = "logical-link";
+ protected static final String PINTERFACE_NODE_TYPE = "p-interface";
+ protected static final String VLAN_NODE_TYPE = "vlan";
+ protected static final String VNFC_NODE_TYPE = "vnfc";
+ protected static final String VSERVER_NODE_TYPE = "vserver";
+ protected static final String PSERVER_NODE_TYPE = "pserver";
+ protected static final String PNF_NODE_TYPE = "pnf";
+ protected static final String NOS_SERVER_NODE_TYPE = "nos-server";
+
+ private static Map<String, Map> map;
+ private static Map<String, Boolean> pair;
+
+ public MigrateInMaintDefaultToFalse(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions, setInMaintToFalse(), false);
+ }
+
+ private static Map<String, Map> setInMaintToFalse(){
+ map = new HashMap<>();
+ pair = new HashMap<>();
+
+ pair.put("in-maint", false);
+
+ map.put("generic-vnf", pair);
+ map.put("l-interface", pair);
+ map.put("lag-interface", pair);
+ map.put("logical-link", pair);
+ map.put("p-interface", pair);
+ map.put("vlan", pair);
+ map.put("vnfc", pair);
+ map.put("vserver", pair);
+ map.put("pserver", pair);
+ map.put("pnf", pair);
+ map.put("nos-server", pair);
+
+ return map;
+ }
+
+ @Override
+ public Status getStatus() {
+ return Status.SUCCESS;
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{VNF_NODE_TYPE,LINTERFACE_NODE_TYPE,LAG_INTERFACE_NODE_TYPE,LOGICAL_LINK_NODE_TYPE,PINTERFACE_NODE_TYPE,VLAN_NODE_TYPE,VNFC_NODE_TYPE,VSERVER_NODE_TYPE,PSERVER_NODE_TYPE,PNF_NODE_TYPE,NOS_SERVER_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInMaintDefaultToFalse";
+ }
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantId.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantId.java
new file mode 100644
index 0000000..1244c59
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelInvariantId.java
@@ -0,0 +1,85 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+/*-
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+
+import org.janusgraph.core.Cardinality;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.PropertyMigrator;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+import java.util.Optional;
+
+
+@MigrationPriority(19)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateInstanceGroupModelInvariantId extends PropertyMigrator {
+
+ private static final String INSTANCE_GROUP_NODE_TYPE = "instance-group";
+ private static final String INSTANCE_GROUP_MODEL_INVARIANT_ID_PROPERTY = "model-invariant-id";
+ private static final String INSTANCE_GROUP_MODEL_INVARIANT_ID_LOCAL_PROPERTY = "model-invariant-id-local";
+
+ public MigrateInstanceGroupModelInvariantId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.initialize(INSTANCE_GROUP_MODEL_INVARIANT_ID_PROPERTY, INSTANCE_GROUP_MODEL_INVARIANT_ID_LOCAL_PROPERTY,
+ String.class, Cardinality.SINGLE);
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{this.INSTANCE_GROUP_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInstanceGroupModelInvariantId";
+ }
+
+ @Override
+ public boolean isIndexed() {
+ return true;
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionId.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionId.java
new file mode 100644
index 0000000..64341ba
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupModelVersionId.java
@@ -0,0 +1,84 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+/*-
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+
+import java.util.Optional;
+import org.janusgraph.core.Cardinality;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.PropertyMigrator;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(19)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateInstanceGroupModelVersionId extends PropertyMigrator {
+
+ private static final String INSTANCE_GROUP_NODE_TYPE = "instance-group";
+ private static final String INSTANCE_GROUP_MODEL_VERSION_ID_PROPERTY = "model-version-id";
+ private static final String INSTANCE_GROUP_MODEL_VERSION_ID_LOCAL_PROPERTY = "model-version-id-local";
+
+ public MigrateInstanceGroupModelVersionId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.initialize(INSTANCE_GROUP_MODEL_VERSION_ID_PROPERTY, INSTANCE_GROUP_MODEL_VERSION_ID_LOCAL_PROPERTY,
+ String.class, Cardinality.SINGLE);
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{this.INSTANCE_GROUP_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInstanceGroupModelVersionId";
+ }
+
+ @Override
+ public boolean isIndexed() {
+ return true;
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupSubType.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupSubType.java
new file mode 100644
index 0000000..6823da8
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupSubType.java
@@ -0,0 +1,64 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.PropertyMigrator;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+
+import org.janusgraph.core.Cardinality;
+import org.onap.aai.setup.SchemaVersions;
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateInstanceGroupSubType extends PropertyMigrator{
+
+ protected static final String SUB_TYPE_PROPERTY = "sub-type";
+ protected static final String INSTANCE_GROUP_ROLE_PROPERTY = "instance-group-role";
+ protected static final String INSTANCE_GROUP_NODE_TYPE = "instance-group";
+
+ public MigrateInstanceGroupSubType(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.initialize(SUB_TYPE_PROPERTY , INSTANCE_GROUP_ROLE_PROPERTY, String.class,Cardinality.SINGLE);
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{INSTANCE_GROUP_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInstanceGroupSubType";
+ }
+
+ @Override
+ public boolean isIndexed() {
+ return true;
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupType.java b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupType.java
new file mode 100644
index 0000000..f3cd669
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateInstanceGroupType.java
@@ -0,0 +1,64 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+import java.util.Optional;
+
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.PropertyMigrator;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+
+import org.janusgraph.core.Cardinality;
+import org.onap.aai.setup.SchemaVersions;
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateInstanceGroupType extends PropertyMigrator{
+
+ protected static final String TYPE_PROPERTY = "type";
+ protected static final String INSTANCE_GROUP_TYPE_PROPERTY = "instance-group-type";
+ protected static final String INSTANCE_GROUP_NODE_TYPE = "instance-group";
+
+ public MigrateInstanceGroupType(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.initialize(TYPE_PROPERTY , INSTANCE_GROUP_TYPE_PROPERTY, String.class,Cardinality.SINGLE);
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{INSTANCE_GROUP_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateInstanceGroupType";
+ }
+
+ @Override
+ public boolean isIndexed() {
+ return true;
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateModelVer.java b/src/main/java/org/onap/aai/migration/v13/MigrateModelVer.java
new file mode 100644
index 0000000..7bc9a7d
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateModelVer.java
@@ -0,0 +1,229 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Migrator;
+import org.onap.aai.migration.Status;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateModelVer extends Migrator{
+
+ protected static final String MODELINVARIANTID = "model-invariant-id";
+ protected static final String MODELVERSIONID = "model-version-id";
+ protected static final String MODELINVARIANTIDLOCAL = "model-invariant-id-local";
+ protected static final String MODELVERSIONIDLOCAL = "model-version-id-local";
+
+ protected static final String MODELVER = "model-ver";
+ protected static final String MODEL = "model";
+
+ protected static final String CONNECTOR_NODETYPE = "connector";
+ protected static final String SERVICEINSTANCE_NODETYPE = "service-instance";
+ protected static final String CONFIGURATION_NODETYPE = "configuration";
+ protected static final String LOGICALLINK_NODETYPE = "logical-link";
+ protected static final String VNFC_NODETYPE = "vnfc";
+ protected static final String L3NETWORK_NODETYPE = "l3-network";
+ protected static final String GENERICVNF_NODETYPE = "generic-vnf";
+ protected static final String PNF_NODETYPE = "pnf";
+ protected static final String VFMODULE_NODETYPE = "vf-module";
+ protected static final String INSTANCEGROUP_NODETYPE = "instance-group";
+ protected static final String ALLOTTEDRESOURCE_NODETYPE = "allotted-resource";
+ protected static final String COLLECTION_NODETYPE = "collection";
+
+ private boolean success = true;
+
+ private static Map<String, String> NODETYPEKEYMAP = new HashMap<String, String>();
+
+ static {
+ NODETYPEKEYMAP.put(CONNECTOR_NODETYPE,"resource-instance-id");
+ NODETYPEKEYMAP.put(SERVICEINSTANCE_NODETYPE,"service-instance-id");
+ NODETYPEKEYMAP.put(CONFIGURATION_NODETYPE, "configuration-id");
+ NODETYPEKEYMAP.put(LOGICALLINK_NODETYPE,"link-name");
+ NODETYPEKEYMAP.put(VNFC_NODETYPE, "vnfc-name");
+ NODETYPEKEYMAP.put(L3NETWORK_NODETYPE, "network-id");
+ NODETYPEKEYMAP.put(GENERICVNF_NODETYPE,"vnf-id");
+ NODETYPEKEYMAP.put(PNF_NODETYPE,"pnf-name");
+ NODETYPEKEYMAP.put(VFMODULE_NODETYPE,"vf-module-id");
+ NODETYPEKEYMAP.put(INSTANCEGROUP_NODETYPE,"id");
+ NODETYPEKEYMAP.put(ALLOTTEDRESOURCE_NODETYPE,"id");
+ NODETYPEKEYMAP.put(COLLECTION_NODETYPE,"collection-id");
+ }
+
+ public MigrateModelVer(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+ @Override
+ public void run() {
+
+ List<Vertex> vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, CONNECTOR_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, CONNECTOR_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, SERVICEINSTANCE_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, SERVICEINSTANCE_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, CONFIGURATION_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, CONFIGURATION_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, LOGICALLINK_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, LOGICALLINK_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VNFC_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, VNFC_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, L3NETWORK_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, L3NETWORK_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, GENERICVNF_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, GENERICVNF_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, PNF_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, PNF_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, VFMODULE_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, VFMODULE_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, INSTANCEGROUP_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, INSTANCEGROUP_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, ALLOTTEDRESOURCE_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, ALLOTTEDRESOURCE_NODETYPE);
+
+ vertextList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, COLLECTION_NODETYPE).has(MODELINVARIANTIDLOCAL).has(MODELVERSIONIDLOCAL).toList();
+ createEdges(vertextList, COLLECTION_NODETYPE);
+ }
+
+ private void createEdges(List<Vertex> sourceVertexList, String nodeTypeString)
+ {
+ int modelVerEdgeCount = 0;
+ int modelVerEdgeErrorCount = 0;
+
+ logger.info("---------- Start Creating an Edge for " + nodeTypeString + " nodes with Model Invariant Id and Model Version Id to the model-ver ----------");
+ Map<String, Vertex> modelVerUriVtxIdMap = new HashMap<String, Vertex>();
+ for (Vertex vertex : sourceVertexList) {
+ String currentValueModelVersionID = null;
+ String currrentValueModelInvariantID = null;
+ try {
+ currentValueModelVersionID = getModelVersionIdNodeValue(vertex);
+ currrentValueModelInvariantID = getModelInvariantIdNodeValue(vertex);
+
+ String uri = String.format("/service-design-and-creation/models/model/%s/model-vers/model-ver/%s", currrentValueModelInvariantID, currentValueModelVersionID);
+ String propertyKey = NODETYPEKEYMAP.get(nodeTypeString);
+ String propertyValue = vertex.value(propertyKey).toString();
+ logger.info("Processing "+nodeTypeString+ " vertex with key "+vertex.value(propertyKey).toString());
+ Vertex modelVerVertex = null;
+
+ if (modelVerUriVtxIdMap.containsKey(uri)){
+ modelVerVertex = modelVerUriVtxIdMap.get(uri);
+ } else {
+ List<Vertex> modelverList = this.engine.asAdmin().getTraversalSource().V().has(MODELINVARIANTID,currrentValueModelInvariantID).has(AAIProperties.NODE_TYPE, MODEL).in()
+ .has(AAIProperties.NODE_TYPE, "model-ver" ).has("aai-uri", uri).toList();
+ if (modelverList != null && !modelverList.isEmpty()) {
+ modelVerVertex = modelverList.get(0);
+ modelVerUriVtxIdMap.put(uri, modelVerVertex);
+ }
+ }
+
+ if (modelVerVertex != null && modelVerVertex.property("model-version-id").isPresent() ) {
+ boolean edgePresent = false;
+ //Check if edge already exists for each of the source vertex
+ List<Vertex> outVertexList = this.engine.asAdmin().getTraversalSource().V(modelVerVertex).in().has("aai-node-type", nodeTypeString).has(propertyKey, propertyValue).toList();
+ Iterator<Vertex> vertexItr = outVertexList.iterator();
+ if (outVertexList != null && !outVertexList.isEmpty() && vertexItr.hasNext()){
+ logger.info("\t Edge already exists from " + nodeTypeString + " node to models-ver with model-invariant-id :" + currrentValueModelInvariantID + " and model-version-id :" + currentValueModelVersionID);
+ edgePresent = true;
+ continue;
+ }
+ // Build edge from vertex to modelVerVertex
+ if (!edgePresent) {
+ this.createPrivateEdge(vertex, modelVerVertex);
+ modelVerEdgeCount++;
+ }
+ } else
+ {
+ modelVerEdgeErrorCount++;
+ logger.info("\t" + MIGRATION_ERROR + "Unable to create edge. No model-ver vertex found with model-invariant-id :" + currrentValueModelInvariantID + " and model-version-id :" + currentValueModelVersionID);
+
+ }
+ } catch (Exception e) {
+ success = false;
+ modelVerEdgeErrorCount++;
+ logger.error("\t" + MIGRATION_ERROR + "encountered exception from " + nodeTypeString + " node when trying to create edge to models-ver with model-invariant-id :" + currrentValueModelInvariantID + " and model-version-id :" + currentValueModelVersionID, e);
+ }
+ }
+
+ logger.info ("\n \n ******* Summary " + nodeTypeString + " Nodes: Finished creating an Edge for " + nodeTypeString + " nodes with Model Invariant Id and Model Version Id to the model-ver Migration ********* \n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of ModelVer edge created from " + nodeTypeString + " nodes: " + modelVerEdgeCount +"\n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of ModelVer edge failed to create the edge from the " + nodeTypeString + " nodes due to error : "+ modelVerEdgeErrorCount +"\n");
+
+
+ }
+ private String getModelInvariantIdNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(MODELINVARIANTIDLOCAL).isPresent()){
+ propertyValue = vertex.value(MODELINVARIANTIDLOCAL).toString();
+ }
+ return propertyValue;
+ }
+
+ private String getModelVersionIdNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(MODELVERSIONIDLOCAL).isPresent()){
+ propertyValue = vertex.value(MODELVERSIONIDLOCAL).toString();
+ }
+ return propertyValue;
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{MODELVER});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateModelVer";
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigratePserverAndPnfEquipType.java b/src/main/java/org/onap/aai/migration/v13/MigratePserverAndPnfEquipType.java
new file mode 100644
index 0000000..6788d7f
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigratePserverAndPnfEquipType.java
@@ -0,0 +1,157 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.Migrator;
+import org.onap.aai.migration.Status;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+@MigrationPriority(20)
+@MigrationDangerRating(2)
+@Enabled
+public class MigratePserverAndPnfEquipType extends Migrator{
+
+ protected static final String EQUIP_TYPE_PROPERTY = "equip-type";
+ protected static final String HOSTNAME_PROPERTY = "hostname";
+ protected static final String PNF_NAME_PROPERTY = "pnf-name";
+ protected static final String PNF_NODE_TYPE = "pnf";
+ protected static final String PSERVER_NODE_TYPE = "pserver";
+ private boolean success = true;
+
+ public MigratePserverAndPnfEquipType(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ }
+
+
+
+ @Override
+ public void run() {
+ int pserverCount = 0;
+ int pnfCount = 0;
+ int pserverErrorCount = 0;
+ int pnfErrorCount = 0;
+ logger.info("---------- Start Updating equip-type for Pserver and Pnf ----------");
+
+ List<Vertex> pserverList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, PSERVER_NODE_TYPE).toList();
+ List<Vertex> pnfList = this.engine.asAdmin().getTraversalSource().V().has(AAIProperties.NODE_TYPE, PNF_NODE_TYPE).toList();
+
+ for (Vertex vertex : pserverList) {
+ String currentValueOfEquipType = null;
+ String hostName = null;
+ try {
+ currentValueOfEquipType = getEquipTypeNodeValue(vertex);
+ hostName = getHostNameNodeValue(vertex);
+ if("Server".equals(currentValueOfEquipType) ||"server".equals(currentValueOfEquipType) ){
+ vertex.property(EQUIP_TYPE_PROPERTY, "SERVER");
+ this.touchVertexProperties(vertex, false);
+ logger.info("changed Pserver equip-type from " + currentValueOfEquipType + " to SERVER having hostname : " + hostName);
+ pserverCount++;
+ }
+ } catch (Exception e) {
+ success = false;
+ pserverErrorCount++;
+ logger.error(MIGRATION_ERROR + "encountered exception for equip-type:" + currentValueOfEquipType + " having hostName :" + hostName, e);
+ }
+ }
+
+ for (Vertex vertex : pnfList) {
+ String currentValueOfEquipType = null;
+ String pnfName = null;
+ try {
+ currentValueOfEquipType = getEquipTypeNodeValue(vertex);
+ pnfName = getPnfNameNodeValue(vertex);
+ if("Switch".equals(currentValueOfEquipType)||"switch".equals(currentValueOfEquipType)){
+ vertex.property(EQUIP_TYPE_PROPERTY, "SWITCH");
+ this.touchVertexProperties(vertex, false);
+ logger.info("changed Pnf equip-type from "+ currentValueOfEquipType +" to SWITCH having pnf-name :" + pnfName);
+ pnfCount++;
+ }
+
+ } catch (Exception e) {
+ success = false;
+ pnfErrorCount++;
+ logger.error(MIGRATION_ERROR + "encountered exception for equip-type:" + currentValueOfEquipType +" having pnf-name : "+ pnfName , e);
+ }
+ }
+
+ logger.info ("\n \n ******* Final Summary Updated equip-type for Pserver and Pnf Migration ********* \n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pservers updated: "+ pserverCount +"\n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pservers failed to update due to error : "+ pserverErrorCount +"\n");
+
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pnf updated: "+ pnfCount +"\n");
+ logger.info(MIGRATION_SUMMARY_COUNT+"Number of Pnf failed to update due to error : "+ pnfErrorCount +"\n");
+
+ }
+
+ private String getEquipTypeNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(EQUIP_TYPE_PROPERTY).isPresent()){
+ propertyValue = vertex.property(EQUIP_TYPE_PROPERTY).value().toString();
+ }
+ return propertyValue;
+ }
+
+ private String getHostNameNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(HOSTNAME_PROPERTY).isPresent()){
+ propertyValue = vertex.property(HOSTNAME_PROPERTY).value().toString();
+ }
+ return propertyValue;
+ }
+
+ private String getPnfNameNodeValue(Vertex vertex) {
+ String propertyValue = "";
+ if(vertex != null && vertex.property(PNF_NAME_PROPERTY).isPresent()){
+ propertyValue = vertex.property(PNF_NAME_PROPERTY).value().toString();
+ }
+ return propertyValue;
+ }
+
+ @Override
+ public Status getStatus() {
+ if (success) {
+ return Status.SUCCESS;
+ } else {
+ return Status.FAILURE;
+ }
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{PSERVER_NODE_TYPE,PNF_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigratePserverAndPnfEquipType";
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantId.java b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantId.java
new file mode 100644
index 0000000..a643842
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelInvariantId.java
@@ -0,0 +1,84 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+/*-
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+
+import org.janusgraph.core.Cardinality;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.PropertyMigrator;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+import java.util.Optional;
+
+
+@MigrationPriority(19)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateVnfcModelInvariantId extends PropertyMigrator {
+
+ private static final String VNFC_NODE_TYPE = "vnfc";
+ private static final String VNFC_MODEL_INVARIANT_ID_PROPERTY = "model-invariant-id";
+ private static final String VNFC_MODEL_INVARIANT_ID_LOCAL_PROPERTY = "model-invariant-id-local";
+
+ public MigrateVnfcModelInvariantId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.initialize(VNFC_MODEL_INVARIANT_ID_PROPERTY, VNFC_MODEL_INVARIANT_ID_LOCAL_PROPERTY, String.class, Cardinality.SINGLE);
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{this.VNFC_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateVnfcModelInvariantId";
+ }
+
+ @Override
+ public boolean isIndexed() {
+ return true;
+ }
+}
diff --git a/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionId.java b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionId.java
new file mode 100644
index 0000000..13cdb80
--- /dev/null
+++ b/src/main/java/org/onap/aai/migration/v13/MigrateVnfcModelVersionId.java
@@ -0,0 +1,83 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.migration.v13;
+/*-
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright (C) 2017 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+
+
+import java.util.Optional;
+import org.janusgraph.core.Cardinality;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.migration.Enabled;
+import org.onap.aai.migration.MigrationDangerRating;
+import org.onap.aai.migration.MigrationPriority;
+import org.onap.aai.migration.PropertyMigrator;
+import org.onap.aai.serialization.db.EdgeSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+
+
+@MigrationPriority(19)
+@MigrationDangerRating(2)
+@Enabled
+public class MigrateVnfcModelVersionId extends PropertyMigrator {
+
+ private static final String VNFC_NODE_TYPE = "vnfc";
+ private static final String VNFC_MODEL_VERSION_ID_PROPERTY = "model-version-id";
+ private static final String VNFC_MODEL_VERSION_ID_LOCAL_PROPERTY = "model-version-id-local";
+
+ public MigrateVnfcModelVersionId(TransactionalGraphEngine engine, LoaderFactory loaderFactory, EdgeIngestor edgeIngestor, EdgeSerializer edgeSerializer, SchemaVersions schemaVersions) {
+ super(engine, loaderFactory, edgeIngestor, edgeSerializer, schemaVersions);
+ this.initialize(VNFC_MODEL_VERSION_ID_PROPERTY, VNFC_MODEL_VERSION_ID_LOCAL_PROPERTY, String.class, Cardinality.SINGLE);
+ }
+
+ @Override
+ public Optional<String[]> getAffectedNodeTypes() {
+ return Optional.of(new String[]{this.VNFC_NODE_TYPE});
+ }
+
+ @Override
+ public String getMigrationName() {
+ return "MigrateVnfcModelVersionId";
+ }
+
+ @Override
+ public boolean isIndexed() {
+ return true;
+ }
+}
diff --git a/src/main/java/org/onap/aai/rest/ExceptionHandler.java b/src/main/java/org/onap/aai/rest/ExceptionHandler.java
new file mode 100644
index 0000000..14c45da
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/ExceptionHandler.java
@@ -0,0 +1,127 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.sun.istack.SAXParseException2;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.ext.ExceptionMapper;
+import javax.ws.rs.ext.Provider;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * The Class ExceptionHandler.
+ */
+@Provider
+public class ExceptionHandler implements ExceptionMapper<Exception> {
+
+ @Context
+ private HttpServletRequest request;
+
+ @Context
+ private HttpHeaders headers;
+
+ /**
+ * @{inheritDoc}
+ */
+ @Override
+ public Response toResponse(Exception exception) {
+
+ Response response = null;
+ ArrayList<String> templateVars = new ArrayList<String>();
+
+ //the general case is that cxf will give us a WebApplicationException
+ //with a linked exception
+ if (exception instanceof WebApplicationException) {
+ WebApplicationException e = (WebApplicationException) exception;
+ if (e.getCause() != null) {
+ if (e.getCause() instanceof SAXParseException2) {
+ templateVars.add("UnmarshalException");
+ AAIException ex = new AAIException("AAI_4007", exception);
+ response = Response
+ .status(400)
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(headers.getAcceptableMediaTypes(), ex, templateVars))
+ .build();
+ }
+ }
+ } else if (exception instanceof JsonParseException) {
+ //jackson does it differently so we get the direct JsonParseException
+ templateVars.add("JsonParseException");
+ AAIException ex = new AAIException("AAI_4007", exception);
+ response = Response
+ .status(400)
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(headers.getAcceptableMediaTypes(), ex, templateVars))
+ .build();
+ } else if (exception instanceof JsonMappingException) {
+ //jackson does it differently so we get the direct JsonParseException
+ templateVars.add("JsonMappingException");
+ AAIException ex = new AAIException("AAI_4007", exception);
+ response = Response
+ .status(400)
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(headers.getAcceptableMediaTypes(), ex, templateVars))
+ .build();
+ }
+
+ // it didn't get set above, we wrap a general fault here
+ if (response == null) {
+
+ Exception actual_e = exception;
+ if (exception instanceof WebApplicationException) {
+ WebApplicationException e = (WebApplicationException) exception;
+ response = e.getResponse();
+ } else {
+ templateVars.add(request.getMethod());
+ templateVars.add("unknown");
+ AAIException ex = new AAIException("AAI_4000", actual_e);
+ List<MediaType> mediaTypes = headers.getAcceptableMediaTypes();
+ int setError = 0;
+
+ for (MediaType mediaType : mediaTypes) {
+ if (MediaType.APPLICATION_XML_TYPE.isCompatible(mediaType)) {
+ response = Response
+ .status(400)
+ .type(MediaType.APPLICATION_XML_TYPE)
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(headers.getAcceptableMediaTypes(), ex, templateVars))
+ .build();
+ setError = 1;
+ }
+ }
+ if (setError == 0) {
+ response = Response
+ .status(400)
+ .type(MediaType.APPLICATION_JSON_TYPE)
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(headers.getAcceptableMediaTypes(), ex, templateVars))
+ .build();
+ }
+ }
+ }
+ return response;
+ }
+}
diff --git a/src/main/java/org/onap/aai/rest/QueryConsumer.java b/src/main/java/org/onap/aai/rest/QueryConsumer.java
new file mode 100644
index 0000000..85665da
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/QueryConsumer.java
@@ -0,0 +1,217 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import org.onap.aai.concurrent.AaiCallable;
+import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.ModelType;
+import org.onap.aai.rest.dsl.DslQueryProcessor;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.StopWatch;
+import org.onap.aai.rest.db.HttpEntry;
+import org.onap.aai.rest.search.GenericQueryProcessor;
+import org.onap.aai.rest.search.QueryProcessorType;
+import org.onap.aai.restcore.HttpMethod;
+import org.onap.aai.restcore.RESTAPI;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.serialization.queryformats.Format;
+import org.onap.aai.serialization.queryformats.FormatFactory;
+import org.onap.aai.serialization.queryformats.Formatter;
+import org.onap.aai.serialization.queryformats.SubGraphStyle;
+import org.onap.aai.setup.SchemaVersion;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.util.AAIConstants;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.*;
+import javax.ws.rs.core.*;
+import javax.ws.rs.core.Response.Status;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+@Component
+@Path("{version: v[1-9][0-9]*|latest}/dbquery")
+public class QueryConsumer extends RESTAPI {
+
+ /** The introspector factory type. */
+ private ModelType introspectorFactoryType = ModelType.MOXY;
+
+ private QueryProcessorType processorType = QueryProcessorType.LOCAL_GROOVY;
+
+ private static final String TARGET_ENTITY = "DB";
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(QueryConsumer.class);
+
+ private HttpEntry traversalUriHttpEntry;
+
+ private DslQueryProcessor dslQueryProcessor;
+
+ private SchemaVersions schemaVersions;
+
+ private String basePath;
+
+ @Autowired
+ public QueryConsumer(
+ HttpEntry traversalUriHttpEntry,
+ DslQueryProcessor dslQueryProcessor,
+ SchemaVersions schemaVersions,
+ @Value("${schema.uri.base.path}") String basePath
+ ){
+ this.traversalUriHttpEntry = traversalUriHttpEntry;
+ this.dslQueryProcessor = dslQueryProcessor;
+ this.basePath = basePath;
+ this.schemaVersions = schemaVersions;
+ }
+
+
+ @PUT
+ @Consumes({ MediaType.APPLICATION_JSON})
+ @Produces({ MediaType.APPLICATION_JSON})
+ public Response executeQuery(String content, @PathParam("version")String versionParam, @PathParam("uri") @Encoded String uri, @DefaultValue("graphson") @QueryParam("format") String queryFormat,@DefaultValue("no_op") @QueryParam("subgraph") String subgraph, @Context HttpHeaders headers, @Context UriInfo info, @Context HttpServletRequest req){
+ return runner(AAIConstants.AAI_GRAPHADMIN_TIMEOUT_ENABLED,
+ AAIConstants.AAI_GRAPHADMIN_TIMEOUT_APP,
+ AAIConstants.AAI_GRAPHADMIN_TIMEOUT_LIMIT,
+ headers,
+ info,
+ HttpMethod.GET,
+ new AaiCallable<Response>() {
+ @Override
+ public Response process() {
+ return processExecuteQuery(content, versionParam, uri, queryFormat, subgraph, headers, info, req);
+ }
+ }
+ );
+ }
+
+ public Response processExecuteQuery(String content, @PathParam("version")String versionParam, @PathParam("uri") @Encoded String uri, @DefaultValue("graphson") @QueryParam("format") String queryFormat,@DefaultValue("no_op") @QueryParam("subgraph") String subgraph, @Context HttpHeaders headers, @Context UriInfo info, @Context HttpServletRequest req) {
+
+ String methodName = "executeQuery";
+ String sourceOfTruth = headers.getRequestHeaders().getFirst("X-FromAppId");
+ String realTime = headers.getRequestHeaders().getFirst("Real-Time");
+ String queryProcessor = headers.getRequestHeaders().getFirst("QueryProcessor");
+ QueryProcessorType processorType = this.processorType;
+ Response response = null;
+ TransactionalGraphEngine dbEngine = null;
+ try {
+ LoggingContext.save();
+ this.checkQueryParams(info.getQueryParameters());
+ Format format = Format.getFormat(queryFormat);
+ if (queryProcessor != null) {
+ processorType = QueryProcessorType.valueOf(queryProcessor);
+ }
+ SubGraphStyle subGraphStyle = SubGraphStyle.valueOf(subgraph);
+ JsonParser parser = new JsonParser();
+
+ JsonObject input = parser.parse(content).getAsJsonObject();
+
+ JsonElement gremlinElement = input.get("gremlin");
+ JsonElement dslElement = input.get("dsl");
+ String queryURI = "";
+ String gremlin = "";
+ String dsl = "";
+
+ SchemaVersion version = new SchemaVersion(versionParam);
+ DBConnectionType type = this.determineConnectionType(sourceOfTruth, realTime);
+ traversalUriHttpEntry.setHttpEntryProperties(version, type);
+ dbEngine = traversalUriHttpEntry.getDbEngine();
+
+ if (gremlinElement != null) {
+ gremlin = gremlinElement.getAsString();
+ }
+ if (dslElement != null) {
+ dsl = dslElement.getAsString();
+ }
+ GenericQueryProcessor processor = null;
+
+ LoggingContext.targetEntity(TARGET_ENTITY);
+ LoggingContext.targetServiceName(methodName);
+ LoggingContext.startTime();
+ StopWatch.conditionalStart();
+
+ if(!dsl.equals("")){
+ processor = new GenericQueryProcessor.Builder(dbEngine)
+ .queryFrom(dsl, "dsl")
+ .queryProcessor(dslQueryProcessor)
+ .processWith(processorType).create();
+ }else {
+ processor = new GenericQueryProcessor.Builder(dbEngine)
+ .queryFrom(gremlin, "gremlin")
+ .processWith(processorType).create();
+ }
+
+ String result = "";
+ List<Object> vertices = processor.execute(subGraphStyle);
+
+ DBSerializer serializer = new DBSerializer(version, dbEngine, introspectorFactoryType, sourceOfTruth);
+ FormatFactory ff = new FormatFactory(traversalUriHttpEntry.getLoader(), serializer, schemaVersions, basePath);
+
+ Formatter formater = ff.get(format, info.getQueryParameters());
+
+ result = formater.output(vertices).toString();
+
+ double msecs = StopWatch.stopIfStarted();
+ LoggingContext.elapsedTime((long)msecs,TimeUnit.MILLISECONDS);
+ LoggingContext.successStatusFields();
+ LOGGER.info ("Completed");
+
+ response = Response.status(Status.OK)
+ .type(MediaType.APPLICATION_JSON)
+ .entity(result).build();
+
+ } catch (AAIException e) {
+ response = consumerExceptionResponseGenerator(headers, info, HttpMethod.GET, e);
+ } catch (Exception e ) {
+ AAIException ex = new AAIException("AAI_4000", e);
+ response = consumerExceptionResponseGenerator(headers, info, HttpMethod.GET, ex);
+ } finally {
+ LoggingContext.restoreIfPossible();
+ LoggingContext.successStatusFields();
+ if (dbEngine != null) {
+ dbEngine.rollback();
+ }
+
+ }
+
+ return response;
+ }
+
+ public void checkQueryParams(MultivaluedMap<String, String> params) throws AAIException {
+
+ if (params.containsKey("depth") && params.getFirst("depth").matches("\\d+")) {
+ String depth = params.getFirst("depth");
+ Integer i = Integer.parseInt(depth);
+ if (i > 1) {
+ throw new AAIException("AAI_3303");
+ }
+ }
+
+
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/rest/dsl/DslListener.java b/src/main/java/org/onap/aai/rest/dsl/DslListener.java
new file mode 100644
index 0000000..e41a946
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/dsl/DslListener.java
@@ -0,0 +1,314 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest.dsl;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.antlr.v4.runtime.tree.TerminalNode;
+import org.onap.aai.AAIDslBaseListener;
+import org.onap.aai.AAIDslParser;
+import org.onap.aai.edges.EdgeIngestor;
+import org.onap.aai.edges.EdgeRule;
+import org.onap.aai.edges.EdgeRuleQuery;
+import org.onap.aai.edges.exceptions.AmbiguousRuleChoiceException;
+import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The Class DslListener.
+ */
+public class DslListener extends AAIDslBaseListener {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DslQueryProcessor.class);
+ private final EdgeIngestor edgeRules;
+
+ //TODO Use StringBuilder to build the query than concat
+ String query = "";
+
+ Map<Integer, String> unionMap = new HashMap<>();
+ Map<String, String> flags = new HashMap<>();
+
+ String currentNode = "";
+ String prevsNode = "";
+ int commas = 0;
+
+ int unionKey = 0;
+ int unionMembers = 0;
+ boolean isUnionBeg = false;
+ boolean isUnionTraversal = false;
+
+ boolean isTraversal = false;
+ boolean isWhereTraversal = false;
+ String whereTraversalNode = "";
+
+ String limitQuery = "";
+ boolean isNot = false;
+
+ /**
+ * Instantiates a new DslListener.
+ */
+ @Autowired
+ public DslListener(EdgeIngestor edgeIngestor) {
+ this.edgeRules = edgeIngestor;
+ }
+
+ @Override
+ public void enterAaiquery(AAIDslParser.AaiqueryContext ctx) {
+ query += "builder";
+ }
+
+ @Override
+ public void enterDslStatement(AAIDslParser.DslStatementContext ctx) {
+ // LOGGER.info("Statement Enter"+ctx.getText());
+ /*
+ * This block of code is entered for every query statement
+ */
+ if (isUnionBeg) {
+ isUnionBeg = false;
+ isUnionTraversal = true;
+
+ } else if (unionMembers > 0) {
+ unionMembers--;
+ query += ",builder.newInstance()";
+ isUnionTraversal = true;
+ }
+
+ }
+
+ @Override
+ public void exitDslStatement(AAIDslParser.DslStatementContext ctx) {
+ /*
+ * Nothing to be done here for now
+ * LOGGER.info("Statement Exit"+ctx.getText());
+ */
+ }
+
+ @Override
+ public void exitAaiquery(AAIDslParser.AaiqueryContext ctx) {
+ /*
+ * dedup is by default for all queries If the query has limit in it
+ * include this as well LOGGER.info("Statement Exit"+ctx.getText());
+ */
+
+ query += ".cap('x').unfold().dedup()" + limitQuery;
+ }
+
+ /*
+ * TODO: The contexts are not inherited from a single parent in AAIDslParser
+ * Need to find a way to do that
+ */
+ @Override
+ public void enterSingleNodeStep(AAIDslParser.SingleNodeStepContext ctx) {
+
+ prevsNode = currentNode;
+ currentNode = ctx.NODE().getText();
+
+ this.generateQuery();
+ if (ctx.STORE() != null && ctx.STORE().getText().equals("*")) {
+ flags.put(currentNode, "store");
+ }
+
+ }
+
+ @Override
+ public void enterSingleQueryStep(AAIDslParser.SingleQueryStepContext ctx) {
+
+ prevsNode = currentNode;
+ currentNode = ctx.NODE().getText();
+ this.generateQuery();
+
+ if (ctx.STORE() != null && ctx.STORE().getText().equals("*")) {
+ flags.put(currentNode, "store");
+ }
+ }
+
+ @Override
+ public void enterMultiQueryStep(AAIDslParser.MultiQueryStepContext ctx) {
+
+ prevsNode = currentNode;
+ currentNode = ctx.NODE().getText();
+ this.generateQuery();
+
+ if (ctx.STORE() != null && ctx.STORE().getText().equals("*")) {
+ flags.put(currentNode, "store");
+ }
+
+ }
+
+ /*
+ * Generates the QueryBuilder syntax for the dsl query
+ */
+ private void generateQuery() {
+ String edgeType = "";
+
+ if (isUnionTraversal || isTraversal || isWhereTraversal) {
+ String previousNode = prevsNode;
+ if (isUnionTraversal) {
+ previousNode = unionMap.get(unionKey);
+ isUnionTraversal = false;
+ }
+
+ EdgeRuleQuery edgeRuleQuery = new EdgeRuleQuery.Builder(previousNode, currentNode).build();
+ EdgeRule edgeRule = null;
+
+ try {
+ edgeRule = edgeRules.getRule(edgeRuleQuery);
+ } catch (EdgeRuleNotFoundException | AmbiguousRuleChoiceException e) {
+ }
+
+ if (edgeRule == null) {
+ edgeType = "EdgeType.COUSIN";
+ } else if ("none".equalsIgnoreCase(edgeRule.getContains())){
+ edgeType = "EdgeType.COUSIN";
+ }else {
+ edgeType = "EdgeType.TREE";
+ }
+
+ query += ".createEdgeTraversal(" + edgeType + ", '" + previousNode + "','" + currentNode + "')";
+
+ }
+
+ else
+ query += ".getVerticesByProperty('aai-node-type', '" + currentNode + "')";
+ }
+
+ @Override
+ public void exitSingleNodeStep(AAIDslParser.SingleNodeStepContext ctx) {
+
+ generateExitStep();
+ }
+
+ @Override
+ public void exitSingleQueryStep(AAIDslParser.SingleQueryStepContext ctx) {
+ generateExitStep();
+ }
+
+ @Override
+ public void exitMultiQueryStep(AAIDslParser.MultiQueryStepContext ctx) {
+ generateExitStep();
+
+ }
+
+ private void generateExitStep() {
+ if (flags.containsKey(currentNode)) {
+ String storeFlag = flags.get(currentNode);
+ if (storeFlag != null && storeFlag.equals("store"))
+ query += ".store('x')";
+ flags.remove(currentNode);
+ }
+ }
+
+ @Override
+ public void enterUnionQueryStep(AAIDslParser.UnionQueryStepContext ctx) {
+ isUnionBeg = true;
+
+ unionKey++;
+ unionMap.put(unionKey, currentNode);
+ query += ".union(builder.newInstance()";
+
+ List<TerminalNode> commaNodes = ctx.COMMA();
+
+ for (TerminalNode node : commaNodes) {
+ unionMembers++;
+ }
+ }
+
+ @Override
+ public void exitUnionQueryStep(AAIDslParser.UnionQueryStepContext ctx) {
+ isUnionBeg = false;
+ unionMap.remove(unionKey);
+
+ query += ")";
+ unionKey--;
+
+ }
+
+ @Override
+ public void enterFilterTraverseStep(AAIDslParser.FilterTraverseStepContext ctx) {
+ isWhereTraversal = true;
+ whereTraversalNode = currentNode;
+ query += ".where(builder.newInstance()";
+ }
+
+ @Override
+ public void exitFilterTraverseStep(AAIDslParser.FilterTraverseStepContext ctx) {
+ query += ")";
+ isWhereTraversal = false;
+ currentNode = whereTraversalNode;
+ }
+
+ @Override
+ public void enterFilterStep(AAIDslParser.FilterStepContext ctx) {
+ if (ctx.NOT() != null && ctx.NOT().getText().equals("!"))
+ isNot = true;
+
+ List<TerminalNode> nodes = ctx.KEY();
+ String key = ctx.KEY(0).getText();
+
+ if (isNot) {
+ query += ".getVerticesExcludeByProperty(";
+ isNot = false;
+ } else
+ query += ".getVerticesByProperty(";
+
+ if (nodes.size() == 2) {
+ query += key + "," + ctx.KEY(1).getText();
+ query += ")";
+ }
+
+ if (nodes.size() > 2) {
+
+ for (TerminalNode node : nodes) {
+ if (node.getText().equals(key))
+ continue;
+
+ query += key + "," + node.getText();
+ query += ")";
+ }
+
+ }
+
+ }
+
+ @Override
+ public void exitFilterStep(AAIDslParser.FilterStepContext ctx) {
+ // For now do nothing
+ }
+
+ @Override
+ public void enterTraverseStep(AAIDslParser.TraverseStepContext ctx) {
+ isTraversal = true;
+ }
+
+ @Override
+ public void exitTraverseStep(AAIDslParser.TraverseStepContext ctx) {
+ isTraversal = false;
+ }
+
+ @Override
+ public void enterLimitStep(AAIDslParser.LimitStepContext ctx) {
+ String value = ctx.NODE().getText();
+ limitQuery += ".limit(" + value + ")";
+ }
+}
diff --git a/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java b/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java
new file mode 100644
index 0000000..582f0ea
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/dsl/DslQueryProcessor.java
@@ -0,0 +1,85 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest.dsl;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.antlr.v4.runtime.CharStreams;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.ParseTreeWalker;
+import org.onap.aai.AAIDslLexer;
+import org.onap.aai.AAIDslParser;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * The Class DslQueryProcessor.
+ */
+public class DslQueryProcessor {
+
+ private static final EELFLogger LOGGER = EELFManager.getInstance().getLogger(DslQueryProcessor.class);
+
+ private DslListener dslListener;
+
+ @Autowired
+ public DslQueryProcessor(DslListener dslListener){
+ this.dslListener = dslListener;
+ }
+
+ public String parseAaiQuery(String aaiQuery) {
+ try {
+ // Create a input stream that reads our string
+ InputStream stream = new ByteArrayInputStream(aaiQuery.getBytes(StandardCharsets.UTF_8));
+
+ // Create a lexer from the input CharStream
+ AAIDslLexer lexer = new AAIDslLexer(CharStreams.fromStream(stream, StandardCharsets.UTF_8));
+
+ // Get a list of tokens pulled from the lexer
+ CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+
+ // Parser that feeds off of the tokens buffer
+ AAIDslParser parser = new AAIDslParser(tokens);
+
+ // Specify our entry point
+ ParseTree ptree = parser.aaiquery();
+ LOGGER.info("QUERY-interim" + ptree.toStringTree(parser));
+
+ // Walk it and attach our listener
+ ParseTreeWalker walker = new ParseTreeWalker();
+ walker.walk(dslListener, ptree);
+ LOGGER.info("Final QUERY" + dslListener.query);
+
+ /*
+ * TODO - Visitor patternQueryDslVisitor visitor = new
+ * QueryDslVisitor(); String query = visitor.visit(ptree);
+ *
+ */
+ return dslListener.query;
+ } catch (Exception e) {
+ LOGGER.error("Error while processing the query"+e.getMessage());
+ }
+ return "";
+ }
+}
diff --git a/src/main/java/org/onap/aai/rest/search/GenericQueryProcessor.java b/src/main/java/org/onap/aai/rest/search/GenericQueryProcessor.java
new file mode 100644
index 0000000..2431d11
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/search/GenericQueryProcessor.java
@@ -0,0 +1,226 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest.search;
+
+import org.apache.tinkerpop.gremlin.process.traversal.P;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.javatuples.Pair;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.rest.dsl.DslQueryProcessor;
+import org.onap.aai.restcore.search.GroovyQueryBuilderSingleton;
+import org.onap.aai.restcore.util.URITools;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.serialization.queryformats.SubGraphStyle;
+
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.MultivaluedMap;
+import java.io.FileNotFoundException;
+import java.net.URI;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public abstract class GenericQueryProcessor {
+
+ protected final Optional<URI> uri;
+ protected final MultivaluedMap<String, String> queryParams;
+ protected final Optional<Collection<Vertex>> vertices;
+ protected static Pattern p = Pattern.compile("query/(.*+)");
+ protected Optional<String> gremlin;
+ protected final TransactionalGraphEngine dbEngine;
+ protected static GroovyQueryBuilderSingleton queryBuilderSingleton = GroovyQueryBuilderSingleton.getInstance();
+ protected final boolean isGremlin;
+ protected Optional<DslQueryProcessor> dslQueryProcessorOptional;
+ /* dsl parameters to store dsl query and to check
+ * if this is a DSL request
+ */
+ protected Optional<String> dsl;
+ protected final boolean isDsl ;
+
+ protected GenericQueryProcessor(Builder builder) {
+ this.uri = builder.getUri();
+ this.dbEngine = builder.getDbEngine();
+ this.vertices = builder.getVertices();
+ this.gremlin = builder.getGremlin();
+ this.isGremlin = builder.isGremlin();
+ this.dsl = builder.getDsl();
+ this.isDsl = builder.isDsl();
+ this.dslQueryProcessorOptional = builder.getDslQueryProcessor();
+
+ if (uri.isPresent()) {
+ queryParams = URITools.getQueryMap(uri.get());
+ } else {
+ queryParams = new MultivaluedHashMap<>();
+ }
+ }
+
+ protected abstract GraphTraversal<?,?> runQuery(String query, Map<String, Object> params);
+
+ protected List<Object> processSubGraph(SubGraphStyle style, GraphTraversal<?,?> g) {
+ final List<Object> resultVertices = new Vector<>();
+ g.store("y");
+
+ if (SubGraphStyle.prune.equals(style) || SubGraphStyle.star.equals(style)) {
+ g.barrier().bothE();
+ if (SubGraphStyle.prune.equals(style)) {
+ g.where(__.otherV().where(P.within("y")));
+ }
+ g.dedup().subgraph("subGraph").cap("subGraph").map(x -> (Graph)x.get()).next().traversal().V().forEachRemaining(x -> {
+ resultVertices.add(x);
+ });
+ } else {
+ resultVertices.addAll(g.toList());
+ }
+ return resultVertices;
+ }
+
+ public List<Object> execute(SubGraphStyle style) throws FileNotFoundException, AAIException {
+ final List<Object> resultVertices;
+
+ Pair<String, Map<String, Object>> tuple = this.createQuery();
+ String query = tuple.getValue0();
+ Map<String, Object> params = tuple.getValue1();
+
+ if (query.equals("") && (vertices.isPresent() && vertices.get().isEmpty())) {
+ //nothing to do, just exit
+ return new ArrayList<>();
+ }
+ GraphTraversal<?,?> g = this.runQuery(query, params);
+
+ resultVertices = this.processSubGraph(style, g);
+
+ return resultVertices;
+ }
+
+ protected Pair<String, Map<String, Object>> createQuery() throws AAIException {
+ Map<String, Object> params = new HashMap<>();
+ String query = "";
+ if (this.isGremlin) {
+ query = gremlin.get();
+
+ }else if (this.isDsl) {
+ String dslUserQuery = dsl.get();
+ if(dslQueryProcessorOptional.isPresent()){
+ String dslQuery = dslQueryProcessorOptional.get().parseAaiQuery(dslUserQuery);
+ query = queryBuilderSingleton.executeTraversal(dbEngine, dslQuery, params);
+ String startPrefix = "g.V()";
+ query = startPrefix + query;
+ }
+ }
+
+ return new Pair<>(query, params);
+ }
+
+ public static class Builder {
+
+ private final TransactionalGraphEngine dbEngine;
+ private Optional<URI> uri = Optional.empty();
+ private Optional<String> gremlin = Optional.empty();
+ private boolean isGremlin = false;
+ private Optional<Collection<Vertex>> vertices = Optional.empty();
+ private QueryProcessorType processorType = QueryProcessorType.GREMLIN_SERVER;
+
+ private Optional<String> dsl = Optional.empty();
+ private boolean isDsl = false;
+ private DslQueryProcessor dslQueryProcessor;
+
+ public Builder(TransactionalGraphEngine dbEngine) {
+ this.dbEngine = dbEngine;
+ }
+
+ public Builder queryFrom(URI uri) {
+ this.uri = Optional.of(uri);
+ this.isGremlin = false;
+ return this;
+ }
+
+ public Builder startFrom(Collection<Vertex> vertices) {
+ this.vertices = Optional.of(vertices);
+ return this;
+ }
+
+ public Builder queryFrom( String query, String queryType) {
+
+ if(queryType.equals("gremlin")){
+ this.gremlin = Optional.of(query);
+ this.isGremlin = true;
+ }
+ if(queryType.equals("dsl")){
+ this.dsl = Optional.of(query);
+ this.isDsl = true;
+ }
+ return this;
+ }
+
+ public Builder processWith(QueryProcessorType type) {
+ this.processorType = type;
+ return this;
+ }
+
+ public Builder queryProcessor(DslQueryProcessor dslQueryProcessor){
+ this.dslQueryProcessor = dslQueryProcessor;
+ return this;
+ }
+
+ public Optional<DslQueryProcessor> getDslQueryProcessor(){
+ return Optional.ofNullable(this.dslQueryProcessor);
+ }
+
+ public TransactionalGraphEngine getDbEngine() {
+ return dbEngine;
+ }
+
+ public Optional<URI> getUri() {
+ return uri;
+ }
+
+ public Optional<String> getGremlin() {
+ return gremlin;
+ }
+
+ public boolean isGremlin() {
+ return isGremlin;
+ }
+
+ public Optional<String> getDsl() {
+ return dsl;
+ }
+
+ public boolean isDsl() {
+ return isDsl;
+ }
+
+ public Optional<Collection<Vertex>> getVertices() {
+ return vertices;
+ }
+
+ public QueryProcessorType getProcessorType() {
+ return processorType;
+ }
+
+ public GenericQueryProcessor create() {
+ return new GroovyShellImpl(this);
+ }
+
+ }
+}
diff --git a/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java b/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java
new file mode 100644
index 0000000..3db4301
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/search/GroovyShellImpl.java
@@ -0,0 +1,45 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest.search;
+
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.onap.aai.restcore.search.GremlinGroovyShellSingleton;
+
+import java.util.Map;
+
+public class GroovyShellImpl extends GenericQueryProcessor {
+
+ protected GroovyShellImpl(Builder builder) {
+ super(builder);
+ }
+
+ @Override
+ protected GraphTraversal<?,?> runQuery(String query, Map<String, Object> params) {
+
+ params.put("g", this.dbEngine.asAdmin().getTraversalSource());
+
+ GremlinGroovyShellSingleton shell = GremlinGroovyShellSingleton.getInstance();
+
+ return shell.executeTraversal(query, params);
+ }
+
+}
+
+
diff --git a/src/main/java/org/onap/aai/rest/search/QueryProcessorType.java b/src/main/java/org/onap/aai/rest/search/QueryProcessorType.java
new file mode 100644
index 0000000..c8e1d14
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/search/QueryProcessorType.java
@@ -0,0 +1,26 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest.search;
+
+public enum QueryProcessorType {
+
+ GREMLIN_SERVER,
+ LOCAL_GROOVY
+}
diff --git a/src/main/java/org/onap/aai/rest/util/EchoResponse.java b/src/main/java/org/onap/aai/rest/util/EchoResponse.java
new file mode 100644
index 0000000..05ff38e
--- /dev/null
+++ b/src/main/java/org/onap/aai/rest/util/EchoResponse.java
@@ -0,0 +1,122 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.rest.util;
+
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.restcore.RESTAPI;
+import org.springframework.stereotype.Component;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * The Class EchoResponse.
+ */
+@Component
+@Path("/util")
+public class EchoResponse extends RESTAPI {
+
+ protected static String authPolicyFunctionName = "util";
+
+ public static final String echoPath = "/util/echo";
+
+ /**
+ * Simple health-check API that echos back the X-FromAppId and X-TransactionId to clients.
+ * If there is a query string, a transaction gets logged into hbase, proving the application is connected to the data store.
+ * If there is no query string, no transacction logging is done to hbase.
+ *
+ * @param headers the headers
+ * @param req the req
+ * @param myAction if exists will cause transaction to be logged to hbase
+ * @return the response
+ */
+ @GET
+ @Produces( { MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
+ @Path("/echo")
+ public Response echoResult(@Context HttpHeaders headers, @Context HttpServletRequest req,
+ @QueryParam("action") String myAction) {
+ Response response = null;
+
+ AAIException ex = null;
+ String fromAppId = null;
+ String transId = null;
+
+ try {
+ fromAppId = getFromAppId(headers );
+ transId = getTransId(headers);
+ } catch (AAIException e) {
+ ArrayList<String> templateVars = new ArrayList<String>();
+ templateVars.add("PUT uebProvider");
+ templateVars.add("addTopic");
+ return Response
+ .status(e.getErrorObject().getHTTPResponseCode())
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(headers.getAcceptableMediaTypes(), e, templateVars))
+ .build();
+ }
+
+ try {
+
+ HashMap<AAIException, ArrayList<String>> exceptionList = new HashMap<AAIException, ArrayList<String>>();
+
+ ArrayList<String> templateVars = new ArrayList<String>();
+ templateVars.add(fromAppId);
+ templateVars.add(transId);
+
+ exceptionList.put(new AAIException("AAI_0002", "OK"), templateVars);
+
+ response = Response.status(Status.OK)
+ .entity(ErrorLogHelper.getRESTAPIInfoResponse(
+ headers.getAcceptableMediaTypes(), exceptionList))
+ .build();
+
+ } catch (Exception e) {
+ ex = new AAIException("AAI_4000", e);
+ ArrayList<String> templateVars = new ArrayList<String>();
+ templateVars.add(Action.GET.name());
+ templateVars.add(fromAppId +" "+transId);
+
+ response = Response
+ .status(Status.INTERNAL_SERVER_ERROR)
+ .entity(ErrorLogHelper.getRESTAPIErrorResponse(
+ headers.getAcceptableMediaTypes(), ex,
+ templateVars)).build();
+
+ } finally {
+ if (ex != null) {
+ ErrorLogHelper.logException(ex);
+ }
+
+ }
+
+ return response;
+ }
+
+}
diff --git a/src/main/java/org/onap/aai/schema/GenTester.java b/src/main/java/org/onap/aai/schema/GenTester.java
new file mode 100644
index 0000000..812c7b0
--- /dev/null
+++ b/src/main/java/org/onap/aai/schema/GenTester.java
@@ -0,0 +1,162 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.schema;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.schema.JanusGraphManagement;
+import org.onap.aai.dbgen.SchemaGenerator;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.logging.ErrorLogHelper;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.onap.aai.util.AAIConfig;
+import org.onap.aai.util.AAIConstants;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.util.Properties;
+import java.util.UUID;
+
+
+public class GenTester {
+
+ private static EELFLogger LOGGER;
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args) {
+
+ JanusGraph graph = null;
+ System.setProperty("aai.service.name", GenTester.class.getSimpleName());
+ // Set the logging file properties to be used by EELFManager
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ LOGGER = EELFManager.getInstance().getLogger(GenTester.class);
+ boolean addDefaultCR = true;
+
+ LoggingContext.init();
+ LoggingContext.component("DBGenTester");
+ LoggingContext.partnerName("AAI-TOOLS");
+ LoggingContext.targetEntity("AAI");
+ LoggingContext.requestId(UUID.randomUUID().toString());
+ LoggingContext.serviceName("AAI");
+ LoggingContext.targetServiceName("main");
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ try {
+ AAIConfig.init();
+ if (args != null && args.length > 0 ){
+ if( "genDbRulesOnly".equals(args[0]) ){
+ ErrorLogHelper.logError("AAI_3100",
+ " This option is no longer supported. What was in DbRules is now derived from the OXM files. ");
+ return;
+ }
+ else if ( "GEN_DB_WITH_NO_SCHEMA".equals(args[0]) ){
+ // Note this is done to create an empty DB with no Schema so that
+ // an HBase copyTable can be used to set up a copy of the db.
+ String imsg = " ---- NOTE --- about to load a graph without doing any schema processing (takes a little while) -------- ";
+ System.out.println(imsg);
+ LOGGER.info(imsg);
+ graph = AAIGraph.getInstance().getGraph();
+
+ if( graph == null ){
+ ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph.");
+ return;
+ }
+ else {
+ String amsg = "Successfully loaded a JanusGraph graph without doing any schema work. ";
+ System.out.println(amsg);
+ LOGGER.auditEvent(amsg);
+ return;
+ }
+ } else if ("GEN_DB_WITH_NO_DEFAULT_CR".equals(args[0])) {
+ addDefaultCR = false;
+ }
+ else {
+ ErrorLogHelper.logError("AAI_3000", "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. ");
+
+ String emsg = "Unrecognized argument passed to GenTester.java: [" + args[0] + "]. ";
+ System.out.println(emsg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ LOGGER.error(emsg);
+
+ emsg = "Either pass no argument for normal processing, or use 'GEN_DB_WITH_NO_SCHEMA'.";
+ System.out.println(emsg);
+ LOGGER.error(emsg);
+
+ return;
+ }
+ }
+
+ //AAIConfig.init();
+ ErrorLogHelper.loadProperties();
+ String imsg = " ---- NOTE --- about to open graph (takes a little while)--------;";
+ System.out.println(imsg);
+ LOGGER.info(imsg);
+ graph = AAIGraph.getInstance().getGraph();
+
+ if( graph == null ){
+ ErrorLogHelper.logError("AAI_5102", "Error creating JanusGraph graph. ");
+ return;
+ }
+
+ // Load the propertyKeys, indexes and edge-Labels into the DB
+ JanusGraphManagement graphMgt = graph.openManagement();
+
+ imsg = "-- Loading new schema elements into JanusGraph --";
+ System.out.println(imsg);
+ LOGGER.info(imsg);
+ SchemaGenerator.loadSchemaIntoJanusGraph(graph, graphMgt, null);
+ } catch(Exception ex) {
+ ErrorLogHelper.logError("AAI_4000", ex.getMessage());
+ }
+
+
+ if( graph != null ){
+ String imsg = "-- graph commit";
+ System.out.println(imsg);
+ LOGGER.info(imsg);
+ graph.tx().commit();
+
+ imsg = "-- graph shutdown ";
+ System.out.println(imsg);
+ LOGGER.info(imsg);
+ graph.close();
+ }
+
+ LOGGER.auditEvent("-- all done, if program does not exit, please kill.");
+ System.exit(0);
+ }
+
+}
+
diff --git a/src/main/java/org/onap/aai/service/AuthorizationService.java b/src/main/java/org/onap/aai/service/AuthorizationService.java
new file mode 100644
index 0000000..d2597d0
--- /dev/null
+++ b/src/main/java/org/onap/aai/service/AuthorizationService.java
@@ -0,0 +1,109 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.service;
+
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.eclipse.jetty.util.security.Password;
+import org.onap.aai.Profiles;
+import org.onap.aai.util.AAIConstants;
+import org.springframework.context.annotation.Profile;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+
+@Profile(Profiles.ONE_WAY_SSL)
+@Service
+public class AuthorizationService {
+
+ private static final EELFLogger logger = EELFManager.getInstance().getLogger(AuthorizationService.class);
+
+ private final Map<String, String> authorizedUsers = new HashMap<>();
+
+ private static final Base64.Encoder ENCODER = Base64.getEncoder();
+
+ @PostConstruct
+ public void init(){
+
+ String basicAuthFile = getBasicAuthFilePath();
+
+ try(Stream<String> stream = Files.lines(Paths.get(basicAuthFile))){
+ stream.filter(line -> !line.startsWith("#")).forEach(str -> {
+ byte [] bytes = null;
+
+ String usernamePassword = null;
+ String accessType = null;
+
+ try {
+ String [] userAccessType = str.split(",");
+
+ if(userAccessType == null || userAccessType.length != 2){
+ throw new RuntimeException("Please check the realm.properties file as it is not conforming to the basic auth");
+ }
+
+ usernamePassword = userAccessType[0];
+ accessType = userAccessType[1];
+
+ String[] usernamePasswordArray = usernamePassword.split(":");
+
+ if(usernamePasswordArray == null || usernamePasswordArray.length != 3){
+ throw new RuntimeException("Not a valid entry for the realm.properties entry: " + usernamePassword);
+ }
+
+ String username = usernamePasswordArray[0];
+ String password = null;
+
+ if(str.contains("OBF:")){
+ password = usernamePasswordArray[1] + ":" + usernamePasswordArray[2];
+ password = Password.deobfuscate(password);
+ }
+
+ bytes = ENCODER.encode((username + ":" + password).getBytes("UTF-8"));
+
+ authorizedUsers.put(new String(bytes), accessType);
+
+ } catch (UnsupportedEncodingException e)
+ {
+ logger.error("Unable to support the encoding of the file" + basicAuthFile);
+ }
+
+ authorizedUsers.put(new String(ENCODER.encode(bytes)), accessType);
+ });
+ } catch (IOException e) {
+ logger.error("IO Exception occurred during the reading of realm.properties", e);
+ }
+ }
+
+ public boolean checkIfUserAuthorized(String authorization){
+ return authorizedUsers.containsKey(authorization) && "admin".equals(authorizedUsers.get(authorization));
+ }
+
+ public String getBasicAuthFilePath(){
+ return AAIConstants.AAI_HOME_ETC_AUTH + AAIConstants.AAI_FILESEP + "realm.properties";
+ }
+}
diff --git a/src/main/java/org/onap/aai/service/RetiredService.java b/src/main/java/org/onap/aai/service/RetiredService.java
new file mode 100644
index 0000000..5989e31
--- /dev/null
+++ b/src/main/java/org/onap/aai/service/RetiredService.java
@@ -0,0 +1,67 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.service;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.PropertySource;
+import org.springframework.stereotype.Service;
+
+import javax.annotation.PostConstruct;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+@Service
+@PropertySource("classpath:retired.properties")
+@PropertySource(value = "file:${server.local.startpath}/retired.properties")
+public class RetiredService {
+
+ private String retiredPatterns;
+
+ private String retiredAllVersions;
+
+ private List<Pattern> retiredPatternsList;
+ private List<Pattern> retiredAllVersionList;
+
+ @PostConstruct
+ public void initialize(){
+ this.retiredPatternsList = Arrays.stream(retiredPatterns.split(",")).map(Pattern::compile).collect(Collectors.toList());
+ this.retiredAllVersionList = Arrays.stream(retiredAllVersions.split(",")).map(Pattern::compile).collect(Collectors.toList());
+ }
+
+ @Value("${retired.api.pattern.list}")
+ public void setRetiredPatterns(String retiredPatterns){
+ this.retiredPatterns = retiredPatterns;
+ }
+
+ public List<Pattern> getRetiredPatterns(){
+ return retiredPatternsList;
+ }
+
+ @Value("${retired.api.all.versions}")
+ public void setRetiredAllVersions(String retiredPatterns){
+ this.retiredAllVersions = retiredPatterns;
+ }
+
+ public List<Pattern> getRetiredAllVersionList(){
+ return retiredAllVersionList;
+ }
+}
diff --git a/src/main/java/org/onap/aai/util/PositiveNumValidator.java b/src/main/java/org/onap/aai/util/PositiveNumValidator.java
new file mode 100644
index 0000000..ee58f55
--- /dev/null
+++ b/src/main/java/org/onap/aai/util/PositiveNumValidator.java
@@ -0,0 +1,35 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.util;
+
+import com.beust.jcommander.IParameterValidator;
+import com.beust.jcommander.ParameterException;
+
+public class PositiveNumValidator implements IParameterValidator {
+
+ @Override
+ public void validate(String name, String value) throws ParameterException {
+ int num = Integer.parseInt(value);
+
+ if(num < 0) {
+ throw new ParameterException("Parameter " + name + " should be >= 0");
+ }
+ }
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java
new file mode 100644
index 0000000..d9615b0
--- /dev/null
+++ b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotifications.java
@@ -0,0 +1,183 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.util;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.*;
+import org.onap.aai.migration.EventAction;
+import org.onap.aai.migration.NotificationHelper;
+import org.onap.aai.rest.ueb.UEBNotification;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.setup.SchemaVersion;
+import org.slf4j.MDC;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import javax.ws.rs.core.Response.Status;
+
+public class SendDeleteMigrationNotifications {
+
+ protected EELFLogger logger = EELFManager.getInstance().getLogger(SendDeleteMigrationNotifications.class.getSimpleName());
+
+ private String config;
+ private String path;
+ private Set<String> notifyOn;
+ long sleepInMilliSecs;
+ int numToBatch;
+ private String requestId;
+ private EventAction eventAction;
+ private String eventSource;
+
+ protected QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ protected ModelType introspectorFactoryType = ModelType.MOXY;
+ protected Loader loader = null;
+ protected TransactionalGraphEngine engine = null;
+ protected NotificationHelper notificationHelper = null;
+ protected DBSerializer serializer = null;
+ protected final LoaderFactory loaderFactory;
+ protected final SchemaVersions schemaVersions;
+ protected final SchemaVersion version;
+
+ public SendDeleteMigrationNotifications(LoaderFactory loaderFactory, SchemaVersions schemaVersions, String config, String path, Set<String> notifyOn, int sleepInMilliSecs, int numToBatch, String requestId, EventAction eventAction, String eventSource) {
+ System.setProperty("aai.service.name", SendDeleteMigrationNotifications.class.getSimpleName());
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
+
+ MDC.put("logFilenameAppender", SendDeleteMigrationNotifications.class.getSimpleName());
+
+ this.config = config;
+ this.path = path;
+ this.notifyOn = notifyOn;
+ this.sleepInMilliSecs = sleepInMilliSecs;
+ this.numToBatch = numToBatch;
+ this.requestId = requestId;
+ this.eventAction = eventAction;
+ this.eventSource = eventSource;
+ this.loaderFactory = loaderFactory;
+ this.schemaVersions = schemaVersions;
+ this.version = schemaVersions.getDefaultVersion();
+ initGraph();
+
+ initFields();
+
+
+ }
+
+ public void process(String basePath) throws Exception {
+
+ try {
+ Map<Integer, String> deleteDataMap = processFile();
+ int count = 0;
+ for (Map.Entry<Integer, String> entry : deleteDataMap.entrySet()) {
+ logger.info("Processing " + entry.getKey() + " :: Data :: " + entry.getValue());
+ String data = entry.getValue();
+ Introspector obj = null;
+ if (data.contains("#@#")) {
+ String[] splitLine = data.split("#@#");
+ if (splitLine.length == 3) {
+ obj = loader.unmarshal(splitLine[0], splitLine[2]);
+ this.notificationHelper.addDeleteEvent(UUID.randomUUID().toString(), splitLine[0], eventAction,
+ URI.create(splitLine[1]), obj, new HashMap(), basePath);
+ }
+ }
+ count++;
+ if (count >= this.numToBatch) {
+ trigger();
+ logger.info("Triggered " + entry.getKey());
+ count = 0;
+ Thread.sleep(this.sleepInMilliSecs);
+ }
+ }
+ if (count > 0) {
+ trigger();
+ }
+ cleanup();
+ } catch (Exception e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ protected void trigger() throws AAIException {
+ this.notificationHelper.triggerEvents();
+ }
+
+ private Map<Integer,String> processFile() throws IOException {
+ List<String> lines = Files.readAllLines(Paths.get(path));
+ final Map<Integer,String> data = new LinkedHashMap<>();
+ AtomicInteger counter = new AtomicInteger(0);
+ lines.stream().forEach(line -> {
+ if (line.contains("#@#")) {
+ data.put(counter.incrementAndGet(), line);
+ }
+ });
+ return data;
+ }
+
+ protected void cleanup() {
+ logAndPrint("Events sent, closing graph connections");
+ engine.rollback();
+ AAIGraph.getInstance().graphShutdown();
+ logAndPrint("---------- Done ----------");
+ }
+
+ private void initFields() {
+ this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ this.engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+ try {
+ this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.eventSource);
+ } catch (AAIException e) {
+ throw new RuntimeException("could not create serializer", e);
+ }
+ this.notificationHelper = new NotificationHelper(loader, serializer, loaderFactory, schemaVersions, engine, requestId, this.eventSource);
+ }
+
+ protected void initGraph() {
+ System.setProperty("realtime.db.config", this.config);
+ logAndPrint("\n\n---------- Connecting to Graph ----------");
+ AAIGraph.getInstance();
+ logAndPrint("---------- Connection Established ----------");
+ }
+
+ protected void logAndPrint(String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java
new file mode 100644
index 0000000..ad96efe
--- /dev/null
+++ b/src/main/java/org/onap/aai/util/SendDeleteMigrationNotificationsMain.java
@@ -0,0 +1,105 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.util;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.migration.EventAction;
+import org.onap.aai.setup.SchemaVersions;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.util.*;
+
+public class SendDeleteMigrationNotificationsMain {
+
+ public static void main(String[] args) {
+
+ Arrays.asList(args).stream().forEach(System.out::println);
+
+ String requestId = UUID.randomUUID().toString();
+ LoggingContext.init();
+ LoggingContext.partnerName("Migration");
+ LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.component("SendMigrationNotifications");
+ LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.targetServiceName("main");
+ LoggingContext.requestId(requestId);
+ LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+ String basePath = ctx.getEnvironment().getProperty("schema.uri.base.path");
+
+ CommandLineDeleteArgs cArgs = new CommandLineDeleteArgs();
+
+ JCommander jCommander = new JCommander(cArgs, args);
+ jCommander.setProgramName(SendDeleteMigrationNotificationsMain.class.getSimpleName());
+
+ EventAction action = EventAction.valueOf(cArgs.eventAction.toUpperCase());
+
+ SendDeleteMigrationNotifications internal = new SendDeleteMigrationNotifications(loaderFactory, schemaVersions, cArgs.config, cArgs.file, new HashSet<>(cArgs.notifyOn), cArgs.sleepInMilliSecs, cArgs.numToBatch, requestId, action, cArgs.eventSource);
+
+ try {
+ internal.process(basePath);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ AAIGraph.getInstance().graphShutdown();
+ System.exit(0);
+ }
+}
+
+class CommandLineDeleteArgs {
+
+ @Parameter(names = "--help", help = true)
+ public boolean help;
+
+ @Parameter(names = "-c", description = "location of configuration file", required = true)
+ public String config;
+
+ @Parameter(names = "--inputFile", description = "path to input file", required = true)
+ public String file;
+
+ @Parameter (names = "--notifyOn", description = "path to input file")
+ public List<String> notifyOn = new ArrayList<>();
+
+ @Parameter (names = "--sleepInMilliSecs", description = "how long to sleep between sending in seconds", validateWith = PositiveNumValidator.class)
+ public Integer sleepInMilliSecs = 0;
+
+ @Parameter (names = "--numToBatch", description = "how many to batch before sending", validateWith = PositiveNumValidator.class)
+ public Integer numToBatch = 1;
+
+ @Parameter (names = "-a", description = "event action type for dmaap event: CREATE, UPDATE, or DELETE")
+ public String eventAction = EventAction.DELETE.toString();
+
+ @Parameter (names = "--eventSource", description = "source of truth for notification, defaults to DMAAP-LOAD")
+ public String eventSource = "DMAAP-LOAD";
+}
+
+
diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotifications.java b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java
new file mode 100644
index 0000000..577f577
--- /dev/null
+++ b/src/main/java/org/onap/aai/util/SendMigrationNotifications.java
@@ -0,0 +1,189 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.util;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.dbmap.DBConnectionType;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.introspection.*;
+import org.onap.aai.migration.EventAction;
+import org.onap.aai.migration.NotificationHelper;
+import org.onap.aai.serialization.db.DBSerializer;
+import org.onap.aai.serialization.engines.QueryStyle;
+import org.onap.aai.serialization.engines.JanusGraphDBEngine;
+import org.onap.aai.serialization.engines.TransactionalGraphEngine;
+import org.onap.aai.setup.SchemaVersions;
+import org.onap.aai.setup.SchemaVersion;
+import org.slf4j.MDC;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.*;
+
+public class SendMigrationNotifications {
+
+ protected EELFLogger logger = EELFManager.getInstance().getLogger(SendMigrationNotifications.class.getSimpleName());
+
+ private String config;
+ private String path;
+ private Set<String> notifyOn;
+ long sleepInMilliSecs;
+ int numToBatch;
+ private String requestId;
+ private EventAction eventAction;
+ private String eventSource;
+
+ protected QueryStyle queryStyle = QueryStyle.TRAVERSAL;
+ protected ModelType introspectorFactoryType = ModelType.MOXY;
+ protected Loader loader = null;
+ protected TransactionalGraphEngine engine = null;
+ protected NotificationHelper notificationHelper = null;
+ protected DBSerializer serializer = null;
+ protected final LoaderFactory loaderFactory;
+ protected final SchemaVersions schemaVersions;
+ protected final SchemaVersion version;
+
+ public SendMigrationNotifications(LoaderFactory loaderFactory, SchemaVersions schemaVersions, String config, String path, Set<String> notifyOn, int sleepInMilliSecs, int numToBatch, String requestId, EventAction eventAction, String eventSource) {
+ System.setProperty("aai.service.name", SendMigrationNotifications.class.getSimpleName());
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, "migration-logback.xml");
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_ETC_APP_PROPERTIES);
+
+ MDC.put("logFilenameAppender", SendMigrationNotifications.class.getSimpleName());
+
+ this.config = config;
+ this.path = path;
+ this.notifyOn = notifyOn;
+ this.sleepInMilliSecs = sleepInMilliSecs;
+ this.numToBatch = numToBatch;
+ this.requestId = requestId;
+ this.eventAction = eventAction;
+ this.eventSource = eventSource;
+ this.loaderFactory = loaderFactory;
+ this.schemaVersions = schemaVersions;
+ this.version = schemaVersions.getDefaultVersion();
+
+ initGraph();
+
+ initFields();
+ }
+
+ public void process(String basePath) throws Exception {
+
+ Map<String, String> vertexIds = processFile();
+ engine.startTransaction();
+ GraphTraversalSource g = engine.asAdmin().getReadOnlyTraversalSource();
+ List<Vertex> vertexes;
+ URI uri;
+ Vertex v;
+ int count = 0;
+ for (Map.Entry<String, String> entry : vertexIds.entrySet()) {
+ vertexes = g.V(entry.getKey()).toList();
+ if (vertexes == null || vertexes.isEmpty()) {
+ logAndPrint("Vertex " + entry.getKey() + " no longer exists." );
+ continue;
+ } else if (vertexes.size() > 1) {
+ logAndPrint("Vertex " + entry.getKey() + " query returned " + vertexes.size() + " vertexes." );
+ continue;
+ } else {
+ logger.info("Processing " + entry.getKey() + "resource-version " + entry.getValue());
+ v = vertexes.get(0);
+ if (notifyOn.isEmpty() || notifyOn.contains(v.value(AAIProperties.NODE_TYPE).toString())) {
+ if (entry.getValue().equals(v.value(AAIProperties.RESOURCE_VERSION).toString())) {
+ Introspector introspector = serializer.getLatestVersionView(v);
+ uri = this.serializer.getURIForVertex(v, false);
+ this.notificationHelper.addEvent(v, introspector, eventAction, uri, basePath);
+ count++;
+ if (count >= this.numToBatch) {
+ trigger();
+ logger.info("Triggered " + entry.getKey());
+ count = 0;
+ Thread.sleep(this.sleepInMilliSecs);
+ }
+ }
+ }
+ }
+ }
+
+ if (count > 0) {
+ trigger();
+ }
+
+ cleanup();
+ }
+
+ protected void trigger() throws AAIException {
+ this.notificationHelper.triggerEvents();
+ }
+
+ private Map<String, String> processFile() throws IOException {
+ List<String> lines = Files.readAllLines(Paths.get(path));
+ final Map<String,String> vertexIds = new LinkedHashMap<>();
+ lines.stream().forEach(line -> {
+ if (line.contains("_")) {
+ String[] splitLine = line.split("_");
+ if (splitLine.length == 2) {
+ vertexIds.put(splitLine[0], splitLine[1]);
+ }
+ }
+ });
+ return vertexIds;
+ }
+
+ protected void cleanup() {
+ logAndPrint("Events sent, closing graph connections");
+ engine.rollback();
+ AAIGraph.getInstance().graphShutdown();
+ logAndPrint("---------- Done ----------");
+ }
+
+ private void initFields() {
+ this.loader = loaderFactory.createLoaderForVersion(introspectorFactoryType, version);
+ this.engine = new JanusGraphDBEngine(queryStyle, DBConnectionType.REALTIME, loader);
+ try {
+ this.serializer = new DBSerializer(version, this.engine, introspectorFactoryType, this.eventSource);
+ } catch (AAIException e) {
+ throw new RuntimeException("could not create serializer", e);
+ }
+ this.notificationHelper = new NotificationHelper(loader, serializer, loaderFactory, schemaVersions, engine, requestId, this.eventSource);
+ }
+
+ protected void initGraph() {
+ System.setProperty("realtime.db.config", this.config);
+ logAndPrint("\n\n---------- Connecting to Graph ----------");
+ AAIGraph.getInstance();
+ logAndPrint("---------- Connection Established ----------");
+ }
+
+ protected void logAndPrint(String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+
+} \ No newline at end of file
diff --git a/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java
new file mode 100644
index 0000000..29eb1da
--- /dev/null
+++ b/src/main/java/org/onap/aai/util/SendMigrationNotificationsMain.java
@@ -0,0 +1,105 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.util;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import org.onap.aai.dbmap.AAIGraph;
+import org.onap.aai.introspection.LoaderFactory;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.migration.EventAction;
+import org.onap.aai.setup.SchemaVersions;
+import org.springframework.context.annotation.AnnotationConfigApplicationContext;
+
+import java.util.*;
+
+public class SendMigrationNotificationsMain {
+
+ public static void main(String[] args) {
+
+ Arrays.asList(args).stream().forEach(System.out::println);
+
+ String requestId = UUID.randomUUID().toString();
+ LoggingContext.init();
+ LoggingContext.partnerName("Migration");
+ LoggingContext.serviceName(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.component("SendMigrationNotifications");
+ LoggingContext.targetEntity(AAIConstants.AAI_RESOURCES_MS);
+ LoggingContext.targetServiceName("main");
+ LoggingContext.requestId(requestId);
+ LoggingContext.statusCode(LoggingContext.StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(
+ "org.onap.aai.config",
+ "org.onap.aai.setup"
+ );
+
+ LoaderFactory loaderFactory = ctx.getBean(LoaderFactory.class);
+ SchemaVersions schemaVersions = ctx.getBean(SchemaVersions.class);
+ String basePath = ctx.getEnvironment().getProperty("schema.uri.base.path");
+
+ CommandLineArgs cArgs = new CommandLineArgs();
+
+ JCommander jCommander = new JCommander(cArgs, args);
+ jCommander.setProgramName(SendMigrationNotificationsMain.class.getSimpleName());
+
+ EventAction action = EventAction.valueOf(cArgs.eventAction.toUpperCase());
+
+ SendMigrationNotifications internal = new SendMigrationNotifications(loaderFactory, schemaVersions, cArgs.config, cArgs.file, new HashSet<>(cArgs.notifyOn), cArgs.sleepInMilliSecs, cArgs.numToBatch, requestId, action, cArgs.eventSource);
+
+ try {
+ internal.process(basePath);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ AAIGraph.getInstance().graphShutdown();
+ System.exit(0);
+ }
+}
+
+class CommandLineArgs {
+
+ @Parameter(names = "--help", help = true)
+ public boolean help;
+
+ @Parameter(names = "-c", description = "location of configuration file", required = true)
+ public String config;
+
+ @Parameter(names = "--inputFile", description = "path to input file", required = true)
+ public String file;
+
+ @Parameter (names = "--notifyOn", description = "path to input file")
+ public List<String> notifyOn = new ArrayList<>();
+
+ @Parameter (names = "--sleepInMilliSecs", description = "how long to sleep between sending in seconds", validateWith = PositiveNumValidator.class)
+ public Integer sleepInMilliSecs = 0;
+
+ @Parameter (names = "--numToBatch", description = "how many to batch before sending", validateWith = PositiveNumValidator.class)
+ public Integer numToBatch = 1;
+
+ @Parameter (names = "-a", description = "event action type for dmaap event: CREATE, UPDATE, or DELETE")
+ public String eventAction = EventAction.CREATE.toString();
+
+ @Parameter (names = "--eventSource", description = "source of truth for notification, defaults to DMAAP-LOAD")
+ public String eventSource = "DMAAP-LOAD";
+}
+
+
diff --git a/src/main/java/org/onap/aai/util/UniquePropertyCheck.java b/src/main/java/org/onap/aai/util/UniquePropertyCheck.java
new file mode 100644
index 0000000..e96c252
--- /dev/null
+++ b/src/main/java/org/onap/aai/util/UniquePropertyCheck.java
@@ -0,0 +1,288 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.util;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.onap.aai.GraphAdminApp;
+import org.onap.aai.exceptions.AAIException;
+import org.onap.aai.logging.LoggingContext;
+import org.onap.aai.logging.LoggingContext.StatusCode;
+import org.slf4j.MDC;
+
+import com.att.eelf.configuration.Configuration;
+import com.att.eelf.configuration.EELFLogger;
+import com.att.eelf.configuration.EELFManager;
+
+import org.janusgraph.core.JanusGraphFactory;
+import org.janusgraph.core.JanusGraph;
+import org.onap.aai.dbmap.AAIGraphConfig;
+
+public class UniquePropertyCheck {
+
+
+ private static final String FROMAPPID = "AAI-UTILS";
+ private static final String TRANSID = UUID.randomUUID().toString();
+ private static final String COMPONENT = "UniquePropertyCheck";
+
+ /**
+ * The main method.
+ *
+ * @param args the arguments
+ */
+ public static void main(String[] args) {
+
+
+ Properties props = System.getProperties();
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_NAME, AAIConstants.AAI_LOGBACK_PROPS);
+ props.setProperty(Configuration.PROPERTY_LOGGING_FILE_PATH, AAIConstants.AAI_HOME_BUNDLECONFIG);
+ EELFLogger logger = EELFManager.getInstance().getLogger(UniquePropertyCheck.class.getSimpleName());
+
+ LoggingContext.init();
+ LoggingContext.partnerName(FROMAPPID);
+ LoggingContext.serviceName(GraphAdminApp.APP_NAME);
+ LoggingContext.component(COMPONENT);
+ LoggingContext.targetEntity(GraphAdminApp.APP_NAME);
+ LoggingContext.targetServiceName("main");
+ LoggingContext.requestId(TRANSID);
+ LoggingContext.statusCode(StatusCode.COMPLETE);
+ LoggingContext.responseCode(LoggingContext.SUCCESS);
+
+ MDC.put("logFilenameAppender", UniquePropertyCheck.class.getSimpleName());
+
+ if( args == null || args.length != 1 ){
+ String msg = "usage: UniquePropertyCheck propertyName \n";
+ System.out.println(msg);
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.BUSINESS_PROCESS_ERROR);
+ logAndPrint(logger, msg );
+ System.exit(1);
+ }
+ String propertyName = args[0];
+ Graph graph = null;
+
+ try {
+ AAIConfig.init();
+ System.out.println(" ---- NOTE --- about to open graph (takes a little while)--------\n");
+ JanusGraph tGraph = JanusGraphFactory.open(new AAIGraphConfig.Builder(AAIConstants.REALTIME_DB_CONFIG).forService(UniquePropertyCheck.class.getSimpleName()).withGraphType("realtime").buildConfiguration());
+
+ if( tGraph == null ) {
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint(logger, " Error: Could not get JanusGraph ");
+ System.exit(1);
+ }
+
+ graph = tGraph.newTransaction();
+ if( graph == null ){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.AVAILABILITY_TIMEOUT_ERROR);
+ logAndPrint(logger, "could not get graph object in UniquePropertyCheck() \n");
+ System.exit(0);
+ }
+ }
+ catch (AAIException e1) {
+ String msg = "Threw Exception: [" + e1.toString() + "]";
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logAndPrint(logger, msg);
+ System.exit(0);
+ }
+ catch (Exception e2) {
+ String msg = "Threw Exception: [" + e2.toString() + "]";
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.UNKNOWN_ERROR);
+ logAndPrint(logger, msg);
+ System.exit(0);
+ }
+
+ runTheCheckForUniqueness( TRANSID, FROMAPPID, graph, propertyName, logger );
+ System.exit(0);
+
+ }// End main()
+
+
+ /**
+ * Run the check for uniqueness.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param graph the graph
+ * @param propertyName the property name
+ * @param logger the logger
+ * @return the boolean
+ */
+ public static Boolean runTheCheckForUniqueness( String transId, String fromAppId, Graph graph,
+ String propertyName, EELFLogger logger ){
+
+ // Note - property can be found in more than one nodetype
+ // our uniqueness constraints are always across the entire db - so this
+ // tool looks across all nodeTypes that the property is found in.
+ Boolean foundDupesFlag = false;
+
+ HashMap <String,String> valuesAndVidHash = new HashMap <String, String> ();
+ HashMap <String,String> dupeHash = new HashMap <String, String> ();
+
+ int propCount = 0;
+ int dupeCount = 0;
+ Iterator<Vertex> vertItor = graph.traversal().V().has(propertyName);
+ while( vertItor.hasNext() ){
+ propCount++;
+ Vertex v = vertItor.next();
+ String thisVid = v.id().toString();
+ Object val = (v.<Object>property(propertyName)).orElse(null);
+ if( valuesAndVidHash.containsKey(val) ){
+ // We've seen this one before- track it in our dupe hash
+ dupeCount++;
+ if( dupeHash.containsKey(val) ){
+ // This is not the first one being added to the dupe hash for this value
+ String updatedDupeList = dupeHash.get(val) + "|" + thisVid;
+ dupeHash.put(val.toString(), updatedDupeList);
+ }
+ else {
+ // This is the first time we see this value repeating
+ String firstTwoVids = valuesAndVidHash.get(val) + "|" + thisVid;
+ dupeHash.put(val.toString(), firstTwoVids);
+ }
+ }
+ else {
+ valuesAndVidHash.put(val.toString(), thisVid);
+ }
+ }
+
+
+ String info = "\n Found this property [" + propertyName + "] " + propCount + " times in our db.";
+ logAndPrint(logger, info);
+ info = " Found " + dupeCount + " cases of duplicate values for this property.\n\n";
+ logAndPrint(logger, info);
+
+ try {
+ if( ! dupeHash.isEmpty() ){
+ Iterator <?> dupeItr = dupeHash.entrySet().iterator();
+ while( dupeItr.hasNext() ){
+ foundDupesFlag = true;
+ Map.Entry pair = (Map.Entry) dupeItr.next();
+ String dupeValue = pair.getKey().toString();;
+ String vidsStr = pair.getValue().toString();
+ String[] vidArr = vidsStr.split("\\|");
+ logAndPrint(logger, "\n\n -------------- Found " + vidArr.length
+ + " nodes with " + propertyName + " of this value: [" + dupeValue + "]. Node details: ");
+
+ for( int i = 0; i < vidArr.length; i++ ){
+ String vidString = vidArr[i];
+ Long idLong = Long.valueOf(vidString);
+ Vertex tvx = graph.traversal().V(idLong).next();
+ showPropertiesAndEdges( TRANSID, FROMAPPID, tvx, logger );
+ }
+ }
+ }
+ }
+ catch( Exception e2 ){
+ LoggingContext.statusCode(StatusCode.ERROR);
+ LoggingContext.responseCode(LoggingContext.DATA_ERROR);
+ logAndPrint(logger, "Threw Exception: [" + e2.toString() + "]");
+ }
+
+
+ return foundDupesFlag;
+
+ }// end of runTheCheckForUniqueness()
+
+
+ /**
+ * Show properties and edges.
+ *
+ * @param transId the trans id
+ * @param fromAppId the from app id
+ * @param tVert the t vert
+ * @param logger the logger
+ */
+ private static void showPropertiesAndEdges( String transId, String fromAppId, Vertex tVert,
+ EELFLogger logger ){
+
+ if( tVert == null ){
+ logAndPrint(logger, "Null node passed to showPropertiesAndEdges.");
+ }
+ else {
+ String nodeType = "";
+ Object ob = tVert.<String>property("aai-node-type").orElse(null);
+ if( ob == null ){
+ nodeType = "null";
+ }
+ else{
+ nodeType = ob.toString();
+ }
+
+ logAndPrint(logger, " AAINodeType/VtxID for this Node = [" + nodeType + "/" + tVert.id() + "]");
+ logAndPrint(logger, " Property Detail: ");
+ Iterator<VertexProperty<Object>> pI = tVert.properties();
+ while( pI.hasNext() ){
+ VertexProperty<Object> tp = pI.next();
+ Object val = tp.value();
+ logAndPrint(logger, "Prop: [" + tp.key() + "], val = [" + val + "] ");
+ }
+
+ Iterator <Edge> eI = tVert.edges(Direction.BOTH);
+ if( ! eI.hasNext() ){
+ logAndPrint(logger, "No edges were found for this vertex. ");
+ }
+ while( eI.hasNext() ){
+ Edge ed = eI.next();
+ String lab = ed.label();
+ Vertex vtx;
+ if (tVert.equals(ed.inVertex())) {
+ vtx = ed.outVertex();
+ } else {
+ vtx = ed.inVertex();
+ }
+ if( vtx == null ){
+ logAndPrint(logger, " >>> COULD NOT FIND VERTEX on the other side of this edge edgeId = " + ed.id() + " <<< ");
+ }
+ else {
+ String nType = vtx.<String>property("aai-node-type").orElse(null);
+ String vid = vtx.id().toString();
+ logAndPrint(logger, "Found an edge (" + lab + ") from this vertex to a [" + nType + "] node with VtxId = " + vid);
+ }
+ }
+ }
+ } // End of showPropertiesAndEdges()
+
+
+ /**
+ * Log and print.
+ *
+ * @param logger the logger
+ * @param msg the msg
+ */
+ protected static void logAndPrint(EELFLogger logger, String msg) {
+ System.out.println(msg);
+ logger.info(msg);
+ }
+
+}
+
diff --git a/src/main/java/org/onap/aai/web/JerseyConfiguration.java b/src/main/java/org/onap/aai/web/JerseyConfiguration.java
new file mode 100644
index 0000000..436946c
--- /dev/null
+++ b/src/main/java/org/onap/aai/web/JerseyConfiguration.java
@@ -0,0 +1,137 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.web;
+
+import org.glassfish.jersey.filter.LoggingFilter;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletProperties;
+import org.onap.aai.rest.QueryConsumer;
+import org.onap.aai.rest.util.EchoResponse;
+import org.reflections.Reflections;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Profile;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Component;
+
+import javax.annotation.Priority;
+import javax.ws.rs.container.ContainerRequestFilter;
+import javax.ws.rs.container.ContainerResponseFilter;
+import java.util.List;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+@Component
+public class JerseyConfiguration extends ResourceConfig {
+
+ private static final Logger log = Logger.getLogger(JerseyConfiguration.class.getName());
+
+ private Environment env;
+
+ @Autowired
+ public JerseyConfiguration(Environment env) {
+
+ this.env = env;
+
+ register(QueryConsumer.class);
+
+ register(EchoResponse.class);
+
+ //Request Filters
+ registerFiltersForRequests();
+ // Response Filters
+ registerFiltersForResponses();
+
+ property(ServletProperties.FILTER_FORWARD_ON_404, true);
+
+ // Following registers the request headers and response headers
+ // If the LoggingFilter second argument is set to true, it will print response value as well
+ if ("true".equalsIgnoreCase(env.getProperty("aai.request.logging.enabled"))) {
+ register(new LoggingFilter(log, false));
+ }
+ }
+
+ public void registerFiltersForRequests() {
+
+ // Find all the classes within the interceptors package
+ Reflections reflections = new Reflections("org.onap.aai.interceptors");
+ // Filter them based on the clazz that was passed in
+ Set<Class<? extends ContainerRequestFilter>> filters = reflections.getSubTypesOf(ContainerRequestFilter.class);
+
+
+ // Check to ensure that each of the filter has the @Priority annotation and if not throw exception
+ for (Class filterClass : filters) {
+ if (filterClass.getAnnotation(Priority.class) == null) {
+ throw new RuntimeException("Container filter " + filterClass.getName() + " does not have @Priority annotation");
+ }
+ }
+
+ // Turn the set back into a list
+ List<Class<? extends ContainerRequestFilter>> filtersList = filters
+ .stream()
+ .filter(f -> {
+ if (f.isAnnotationPresent(Profile.class)
+ && !env.acceptsProfiles(f.getAnnotation(Profile.class).value())) {
+ return false;
+ }
+ return true;
+ })
+ .collect(Collectors.toList());
+
+ // Sort them by their priority levels value
+ filtersList.sort((c1, c2) -> Integer.valueOf(c1.getAnnotation(Priority.class).value()).compareTo(c2.getAnnotation(Priority.class).value()));
+
+ // Then register this to the jersey application
+ filtersList.forEach(this::register);
+ }
+
+ public void registerFiltersForResponses() {
+
+ // Find all the classes within the interceptors package
+ Reflections reflections = new Reflections("org.onap.aai.interceptors");
+ // Filter them based on the clazz that was passed in
+ Set<Class<? extends ContainerResponseFilter>> filters = reflections.getSubTypesOf(ContainerResponseFilter.class);
+
+
+ // Check to ensure that each of the filter has the @Priority annotation and if not throw exception
+ for (Class filterClass : filters) {
+ if (filterClass.getAnnotation(Priority.class) == null) {
+ throw new RuntimeException("Container filter " + filterClass.getName() + " does not have @Priority annotation");
+ }
+ }
+
+ // Turn the set back into a list
+ List<Class<? extends ContainerResponseFilter>> filtersList = filters.stream()
+ .filter(f -> {
+ if (f.isAnnotationPresent(Profile.class)
+ && !env.acceptsProfiles(f.getAnnotation(Profile.class).value())) {
+ return false;
+ }
+ return true;
+ })
+ .collect(Collectors.toList());
+
+ // Sort them by their priority levels value
+ filtersList.sort((c1, c2) -> Integer.valueOf(c1.getAnnotation(Priority.class).value()).compareTo(c2.getAnnotation(Priority.class).value()));
+
+ // Then register this to the jersey application
+ filtersList.forEach(this::register);
+ }
+}
diff --git a/src/main/java/org/onap/aai/web/LocalHostAccessLog.java b/src/main/java/org/onap/aai/web/LocalHostAccessLog.java
new file mode 100644
index 0000000..4e28562
--- /dev/null
+++ b/src/main/java/org/onap/aai/web/LocalHostAccessLog.java
@@ -0,0 +1,67 @@
+/**
+ * ============LICENSE_START=======================================================
+ * org.onap.aai
+ * ================================================================================
+ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
+ * ================================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============LICENSE_END=========================================================
+ */
+package org.onap.aai.web;
+
+import ch.qos.logback.access.jetty.RequestLogImpl;
+import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.eclipse.jetty.server.handler.RequestLogHandler;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.embedded.EmbeddedServletContainerFactory;
+import org.springframework.boot.context.embedded.jetty.JettyEmbeddedServletContainerFactory;
+import org.springframework.boot.context.embedded.jetty.JettyServerCustomizer;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.Arrays;
+
+@Configuration
+public class LocalHostAccessLog {
+
+ @Bean
+ public EmbeddedServletContainerFactory jettyConfigBean(
+ @Value("${jetty.threadPool.maxThreads:200}") final String maxThreads,
+ @Value("${jetty.threadPool.minThreads:8}") final String minThreads
+ ){
+ JettyEmbeddedServletContainerFactory jef = new JettyEmbeddedServletContainerFactory();
+ jef.addServerCustomizers((JettyServerCustomizer) server -> {
+
+ HandlerCollection handlers = new HandlerCollection();
+
+ Arrays.stream(server.getHandlers()).forEach(handlers::addHandler);
+
+ RequestLogHandler requestLogHandler = new RequestLogHandler();
+ requestLogHandler.setServer(server);
+
+ RequestLogImpl requestLogImpl = new RequestLogImpl();
+ requestLogImpl.setResource("/localhost-access-logback.xml");
+ requestLogImpl.start();
+
+ requestLogHandler.setRequestLog(requestLogImpl);
+ handlers.addHandler(requestLogHandler);
+ server.setHandler(handlers);
+
+ final QueuedThreadPool threadPool = server.getBean(QueuedThreadPool.class);
+ threadPool.setMaxThreads(Integer.valueOf(maxThreads));
+ threadPool.setMinThreads(Integer.valueOf(minThreads));
+ });
+ return jef;
+ }
+}