diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | aai-aaf-auth/pom.xml | 2 | ||||
| -rw-r--r-- | aai-annotations/pom.xml | 2 | ||||
| -rw-r--r-- | aai-auth/pom.xml | 2 | ||||
| -rw-r--r-- | aai-common-docker/aai-common-images/pom.xml | 4 | ||||
| -rw-r--r-- | aai-common-docker/aai-haproxy-image/pom.xml | 4 | ||||
| -rw-r--r-- | aai-common-docker/pom.xml | 2 | ||||
| -rw-r--r-- | aai-core/pom.xml | 2 | ||||
| -rw-r--r-- | aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java | 419 | ||||
| -rw-r--r-- | aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java | 8 | ||||
| -rw-r--r-- | aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java | 3 | ||||
| -rw-r--r-- | aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java | 43 | ||||
| -rw-r--r-- | aai-els-onap-logging/pom.xml | 2 | ||||
| -rw-r--r-- | aai-failover/pom.xml | 2 | ||||
| -rw-r--r-- | aai-parent/pom.xml | 2 | ||||
| -rw-r--r-- | aai-rest/pom.xml | 2 | ||||
| -rw-r--r-- | aai-schema-abstraction/pom.xml | 2 | ||||
| -rw-r--r-- | aai-schema-ingest/pom.xml | 2 | ||||
| -rw-r--r-- | aai-utils/pom.xml | 2 | ||||
| -rw-r--r-- | pom.xml | 2 | ||||
| -rw-r--r-- | version.properties | 2 |
21 files changed, 378 insertions, 134 deletions
@@ -15,4 +15,5 @@ bundleconfig-local/etc/logback.xml .vscode/ .devcontainer/ *.log -aai-core/logs/
\ No newline at end of file +aai-core/logs/ +.classpath
\ No newline at end of file diff --git a/aai-aaf-auth/pom.xml b/aai-aaf-auth/pom.xml index 9e69e603..12ca1b7b 100644 --- a/aai-aaf-auth/pom.xml +++ b/aai-aaf-auth/pom.xml @@ -6,7 +6,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-aaf-auth</artifactId> diff --git a/aai-annotations/pom.xml b/aai-annotations/pom.xml index 60fa4eb3..e8ef6924 100644 --- a/aai-annotations/pom.xml +++ b/aai-annotations/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-annotations</artifactId> diff --git a/aai-auth/pom.xml b/aai-auth/pom.xml index 3922d6cf..ac251a42 100644 --- a/aai-auth/pom.xml +++ b/aai-auth/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-auth</artifactId> diff --git a/aai-common-docker/aai-common-images/pom.xml b/aai-common-docker/aai-common-images/pom.xml index 4cb93653..e15c6f2e 100644 --- a/aai-common-docker/aai-common-images/pom.xml +++ b/aai-common-docker/aai-common-images/pom.xml @@ -25,11 +25,11 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-common-docker</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> </parent> <artifactId>aai-common-images</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <packaging>pom</packaging> <name>aai-aai-common-images</name> <description>Contains dockerfiles for aai-common images (alpine and ubuntu based).</description> diff --git a/aai-common-docker/aai-haproxy-image/pom.xml b/aai-common-docker/aai-haproxy-image/pom.xml index f77f4ef8..c2172309 100644 --- a/aai-common-docker/aai-haproxy-image/pom.xml +++ b/aai-common-docker/aai-haproxy-image/pom.xml @@ -25,11 +25,11 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-common-docker</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> </parent> <artifactId>aai-haproxy-image</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <packaging>pom</packaging> <name>aai-aai-haproxy-image</name> <description>Contains dockerfiles for aai-haproxy image.</description> diff --git a/aai-common-docker/pom.xml b/aai-common-docker/pom.xml index e0e29fe2..80d7ece9 100644 --- a/aai-common-docker/pom.xml +++ b/aai-common-docker/pom.xml @@ -26,7 +26,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> diff --git a/aai-core/pom.xml b/aai-core/pom.xml index 348d4eb0..fb646692 100644 --- a/aai-core/pom.xml +++ b/aai-core/pom.xml @@ -26,7 +26,7 @@ limitations under the License. <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-core</artifactId> diff --git a/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java b/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java index 250c1263..6430df50 100644 --- a/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java +++ b/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java @@ -4,6 +4,8 @@ * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. * ================================================================================ + * Modifications Copyright © 2024 DEUTSCHE TELEKOM AG. + * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -22,21 +24,38 @@ package org.onap.aai.dbgen; import com.google.common.collect.Multimap; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.janusgraph.core.Cardinality; +import org.janusgraph.core.EdgeLabel; +import org.janusgraph.core.JanusGraph; import org.janusgraph.core.Multiplicity; import org.janusgraph.core.PropertyKey; import org.janusgraph.core.schema.ConsistencyModifier; import org.janusgraph.core.schema.JanusGraphIndex; import org.janusgraph.core.schema.JanusGraphManagement; +import org.janusgraph.core.schema.JanusGraphManagement.IndexJobFuture; +import org.janusgraph.core.schema.RelationTypeIndex; +import org.janusgraph.core.schema.SchemaAction; +import org.janusgraph.core.schema.SchemaStatus; +import org.janusgraph.graphdb.database.StandardJanusGraph; +import org.janusgraph.graphdb.database.management.ManagementSystem; +import org.janusgraph.graphdb.database.management.RelationIndexStatusReport; import org.onap.aai.config.SpringContextAware; import org.onap.aai.edges.EdgeIngestor; import org.onap.aai.edges.EdgeRule; @@ -53,6 +72,7 @@ import org.slf4j.LoggerFactory; public class SchemaGenerator { private static final Logger LOGGER = LoggerFactory.getLogger(SchemaGenerator.class); + private static int indexRecoveryRetryCounter = 0; private SchemaGenerator() { } @@ -63,7 +83,9 @@ public class SchemaGenerator { * @param graphMgmt * the graph mgmt */ - public static void loadSchemaIntoJanusGraph(final JanusGraphManagement graphMgmt, String backend) { + public static List<String> loadSchemaIntoJanusGraph(final JanusGraph graph, String backend, boolean dbNotEmpty) { + JanusGraphManagement graphMgmt = graph.openManagement(); + final List<String> elementsToReindex = new ArrayList<>(); try { AAIConfig.init(); @@ -85,121 +107,342 @@ public class SchemaGenerator { // multiplicty definitions depends on which two types of nodes are being // connected. - makeEdgeLabels(graphMgmt); + final Map<String, Introspector> objs = LoaderUtil.getLatestVersion().getAllObjects(); + final Map<String, PropertyKey> seenProps = new HashMap<>(); + + for (Introspector obj : objs.values()) { + createSchemaForObject(graphMgmt, seenProps, obj); + } - Map<String, Introspector> objs = LoaderUtil.getLatestVersion().getAllObjects(); - Map<String, PropertyKey> seenProps = new HashMap<>(); + makeEdgeLabels(graphMgmt, elementsToReindex, dbNotEmpty); - for (Introspector obj : objs.values()) { - for (String propName : obj.getProperties()) { - String dbPropName = propName; - Optional<String> alias = obj.getPropertyMetadata(propName, PropertyMetadata.DB_ALIAS); - if (alias.isPresent()) { - dbPropName = alias.get(); + LOGGER.info("-- About to call graphMgmt commit"); + if (backend != null) { + LOGGER.info("Successfully loaded the schema to {}", backend); + } + + graphMgmt.commit(); + return elementsToReindex; + } + + public static void reindexEdgeIndexes(final JanusGraph graph, Collection<String> edgeLabels) { + graph.tx().rollback(); + if(edgeLabels.isEmpty()) { + LOGGER.info("Nothing to reindex."); + return; + } + + ensureValidIndexState(graph, edgeLabels); + + awaitRelationIndexStatus(graph, edgeLabels,SchemaStatus.REGISTERED); + + LOGGER.info("Attempting to transition indexes in REGISTERED state to ENABLED"); + updateRelationIndexes(graph, edgeLabels, SchemaAction.REINDEX); + + ensureEnabledIndexState(graph, edgeLabels); + } + + /** + * Indices need to be in status REGISTERED or ENABLED to allow reindexing. + * @param graph + * @param edgeLabels + */ + private static void ensureEnabledIndexState(final JanusGraph graph, Collection<String> edgeLabels) { + JanusGraphManagement graphMgmt = graph.openManagement(); + List<String> registeredIndexes = new ArrayList<>(); + for(String label: edgeLabels) { + EdgeLabel relation = graphMgmt.getEdgeLabel(label); + RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label); + SchemaStatus indexStatus = index.getIndexStatus(); + if(indexStatus.equals(SchemaStatus.REGISTERED)) { + LOGGER.info("Detected relation index [{}] that is not yet in ENABLED state", relation.name()); + registeredIndexes.add(label); + } + } + graphMgmt.commit(); + + if(indexRecoveryRetryCounter <= 8 && !registeredIndexes.isEmpty()) { + indexRecoveryRetryCounter++; + LOGGER.info("[{}] indexes not yet in ENABLED state", registeredIndexes.size()); + awaitRelationIndexStatus(graph, registeredIndexes,SchemaStatus.ENABLED); + + ensureEnabledIndexState(graph, edgeLabels); // recursively call to make sure there is no invalid state + } else { + LOGGER.info("All indexes are in ENABLED state, exiting."); + return; + } + } + + /** + * Indices need to be in status REGISTERED or ENABLED to allow reindexing. + * @param graph + * @param edgeLabels + */ + private static void ensureValidIndexState(final JanusGraph graph, Collection<String> edgeLabels) { + JanusGraphManagement graphMgmt = graph.openManagement(); + List<String> installedIndexes = new ArrayList<>(); + List<String> disabledIndexes = new ArrayList<>(); + for(String label: edgeLabels) { + EdgeLabel relation = graphMgmt.getEdgeLabel(label); + RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label); + SchemaStatus indexStatus = index.getIndexStatus(); + if(indexStatus.equals(SchemaStatus.INSTALLED)) { + LOGGER.info("Detected relation index [{}] with invalid status [{}]", relation.name(), indexStatus); + installedIndexes.add(label); + } else if(indexStatus.equals(SchemaStatus.DISABLED)) { + LOGGER.info("Detected relation index [{}] with invalid status [{}]", relation.name(), indexStatus); + disabledIndexes.add(label); + } + } + graphMgmt.commit(); + + if(indexRecoveryRetryCounter <= 300 && (!installedIndexes.isEmpty() || !disabledIndexes.isEmpty())) { + indexRecoveryRetryCounter++; + if(!installedIndexes.isEmpty()) { + LOGGER.info("Attempting to transition indexes in INSTALLED state to REGISTERED"); + updateRelationIndexes(graph, installedIndexes, SchemaAction.REGISTER_INDEX); + awaitRelationIndexStatus(graph, edgeLabels,SchemaStatus.REGISTERED); + } + if(!disabledIndexes.isEmpty()) { + LOGGER.info("Attempting to transition indexes in DISABLED state to ENABLED"); + updateRelationIndexes(graph, disabledIndexes, SchemaAction.ENABLE_INDEX); + awaitRelationIndexStatus(graph, edgeLabels,SchemaStatus.ENABLED); + } + ensureValidIndexState(graph, edgeLabels); // recursively call to make sure there is no invalid state + } else { + return; + } + } + + private static void createSchemaForObject(final JanusGraphManagement graphMgmt, Map<String, PropertyKey> seenProps, + Introspector obj) { + for (String propName : obj.getProperties()) { + String dbPropName = propName; + Optional<String> alias = obj.getPropertyMetadata(propName, PropertyMetadata.DB_ALIAS); + if (alias.isPresent()) { + dbPropName = alias.get(); + } + if (graphMgmt.containsRelationType(dbPropName)) { + LOGGER.debug(" PropertyKey [{}] already existed in the DB. ", dbPropName); + } else { + Class<?> type = obj.getClass(propName); + Cardinality cardinality = Cardinality.SINGLE; + boolean process = false; + if (obj.isListType(propName) && obj.isSimpleGenericType(propName)) { + cardinality = Cardinality.SET; + type = obj.getGenericTypeClass(propName); + process = true; + } else if (obj.isSimpleType(propName)) { + process = true; } - if (graphMgmt.containsRelationType(dbPropName)) { - LOGGER.debug(" PropertyKey [{}] already existed in the DB. ", dbPropName); - } else { - Class<?> type = obj.getClass(propName); - Cardinality cardinality = Cardinality.SINGLE; - boolean process = false; - if (obj.isListType(propName) && obj.isSimpleGenericType(propName)) { - cardinality = Cardinality.SET; - type = obj.getGenericTypeClass(propName); - process = true; - } else if (obj.isSimpleType(propName)) { - process = true; - } - if (process) { - - LOGGER.info("Creating PropertyKey: [{}], [{}], [{}]", dbPropName, type.getSimpleName(), - cardinality); - PropertyKey propK; - if (!seenProps.containsKey(dbPropName)) { - propK = graphMgmt.makePropertyKey(dbPropName).dataType(type).cardinality(cardinality) - .make(); - if (dbPropName.equals("aai-uri")) { - String aai_uri_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false"); - LOGGER.info(" Info: aai_uri_lock_enabled:" + aai_uri_lock_enabled); - if ("true".equals(aai_uri_lock_enabled)) { - LOGGER.info(" Lock is being set for aai-uri Property."); - graphMgmt.setConsistency(propK, ConsistencyModifier.LOCK); - } - } else if (dbPropName.equals("resource-version")) { - String aai_rv_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false"); - LOGGER.info(" Info: aai_rv_lock_enabled:" + aai_rv_lock_enabled); - if ("true".equals(aai_rv_lock_enabled)) { - LOGGER.info(" Lock is being set for resource-version Property."); - graphMgmt.setConsistency(propK, ConsistencyModifier.LOCK); - } + if (process) { + + LOGGER.info("Creating PropertyKey: [{}], [{}], [{}]", dbPropName, type.getSimpleName(), + cardinality); + PropertyKey propertyKey; + if (!seenProps.containsKey(dbPropName)) { + propertyKey = graphMgmt.makePropertyKey(dbPropName).dataType(type).cardinality(cardinality) + .make(); + if (dbPropName.equals("aai-uri")) { + String aai_uri_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false"); + LOGGER.info(" Info: aai_uri_lock_enabled:" + aai_uri_lock_enabled); + if ("true".equals(aai_uri_lock_enabled)) { + LOGGER.info(" Lock is being set for aai-uri Property."); + graphMgmt.setConsistency(propertyKey, ConsistencyModifier.LOCK); + } + } else if (dbPropName.equals("resource-version")) { + String aai_rv_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false"); + LOGGER.info(" Info: aai_rv_lock_enabled:" + aai_rv_lock_enabled); + if ("true".equals(aai_rv_lock_enabled)) { + LOGGER.info(" Lock is being set for resource-version Property."); + graphMgmt.setConsistency(propertyKey, ConsistencyModifier.LOCK); } - seenProps.put(dbPropName, propK); - } else { - propK = seenProps.get(dbPropName); } - if (graphMgmt.containsGraphIndex(dbPropName)) { - LOGGER.debug(" Index [{}] already existed in the DB. ", dbPropName); + seenProps.put(dbPropName, propertyKey); + } else { + propertyKey = seenProps.get(dbPropName); + } + if (graphMgmt.containsGraphIndex(dbPropName)) { + LOGGER.debug(" Index [{}] already existed in the DB. ", dbPropName); + } else { + if (obj.getIndexedProperties().contains(propName)) { + createIndexForProperty(graphMgmt, obj, propName, dbPropName, propertyKey); } else { - if (obj.getIndexedProperties().contains(propName)) { - JanusGraphIndex indexG = null; - if (obj.getUniqueProperties().contains(propName)) { - LOGGER.info("Add Unique index for PropertyKey: [{}]", dbPropName); - indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propK).unique() - .buildCompositeIndex(); - } else { - LOGGER.info("Add index for PropertyKey: [{}]", dbPropName); - indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propK) - .buildCompositeIndex(); - } - if (indexG != null && dbPropName.equals("aai-uri")) { - String aai_uri_lock_enabled = - AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false"); - LOGGER.info(" Info:: aai_uri_lock_enabled:" + aai_uri_lock_enabled); - if ("true".equals(aai_uri_lock_enabled)) { - LOGGER.info("Lock is being set for aai-uri Index."); - graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK); - } - } else if (indexG != null && dbPropName.equals("resource-version")) { - String aai_rv_lock_enabled = - AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false"); - LOGGER.info(" Info:: aai_rv_lock_enabled:" + aai_rv_lock_enabled); - if ("true".equals(aai_rv_lock_enabled)) { - LOGGER.info("Lock is being set for resource-version Index."); - graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK); - } - } - } else { - LOGGER.info("No index added for PropertyKey: [{}]", dbPropName); - } + LOGGER.info("No index added for PropertyKey: [{}]", dbPropName); } } } } } + } - LOGGER.info("-- About to call graphMgmt commit"); - if (backend != null) { - LOGGER.info("Successfully loaded the schema to {}", backend); + private static void createIndexForProperty(final JanusGraphManagement graphMgmt, Introspector obj, String propName, + String dbPropName, PropertyKey propertyKey) { + JanusGraphIndex indexG = null; + if (obj.getUniqueProperties().contains(propName)) { + LOGGER.info("Add Unique index for PropertyKey: [{}]", dbPropName); + indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propertyKey).unique() + .buildCompositeIndex(); + } else { + LOGGER.info("Add index for PropertyKey: [{}]", dbPropName); + indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propertyKey) + .buildCompositeIndex(); + } + if (indexG != null && dbPropName.equals("aai-uri")) { + String aai_uri_lock_enabled = + AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false"); + LOGGER.info(" Info:: aai_uri_lock_enabled:" + aai_uri_lock_enabled); + if ("true".equals(aai_uri_lock_enabled)) { + LOGGER.info("Lock is being set for aai-uri Index."); + graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK); + } + } else if (indexG != null && dbPropName.equals("resource-version")) { + String aai_rv_lock_enabled = + AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false"); + LOGGER.info(" Info:: aai_rv_lock_enabled:" + aai_rv_lock_enabled); + if ("true".equals(aai_rv_lock_enabled)) { + LOGGER.info("Lock is being set for resource-version Index."); + graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK); + } + } + } + + /** + * Debug method to print current index states. + * This can help diagnosing indexes that are stuck in INSTALLED state. + * @param graph + */ + public static void printCurrentRelationIndexStates(JanusGraph graph) { + JanusGraphManagement graphMgmt = graph.openManagement(); + EdgeIngestor edgeIngestor = SpringContextAware.getBean(EdgeIngestor.class); + try { + Set<String> edgeLabels = Optional.ofNullable(edgeIngestor.getAllCurrentRules()) + .map(collectValues(EdgeRule::getLabel)).orElseGet(HashSet::new); + edgeLabels.stream() + .forEach(label -> { + EdgeLabel relation = graphMgmt.getEdgeLabel(label); + RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label); + LOGGER.info("Index state of relation index [{}] is [{}]",label, index.getIndexStatus()); + }); + } catch (EdgeRuleNotFoundException e) { + e.printStackTrace(); + } + } + + /** + * Jointly wait for a set of relation indexes to change status. + * Use this after calling {@link JanusGraphManagement#updateIndex(org.janusgraph.core.schema.Index, SchemaAction)} + * @param graph the JanusGraph + * @param labels the names of the indexes + * @param newStatus the new SchemaStatus + */ + private static void awaitRelationIndexStatus(JanusGraph graph, Collection<String> labels, SchemaStatus newStatus) { + LOGGER.info("Awaiting index status [{}]", newStatus);; + CompletableFuture<RelationIndexStatusReport>[] awaits = labels.stream() + .map(label -> + CompletableFuture.supplyAsync(() -> { + try { + return ManagementSystem + .awaitRelationIndexStatus(graph, label, label) + .status(newStatus) + .call(); + } catch (InterruptedException e) { + e.printStackTrace(); + return null; + } + })) + .toArray(CompletableFuture[]::new); + try { + CompletableFuture.allOf(awaits).get(); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error("Error while waiting for change in relation index status"); + e.printStackTrace(); } + LOGGER.info("Completed waiting for index status [{}]", newStatus); + } + + private static void updateRelationIndexes(JanusGraph graph, Collection<String> labels, SchemaAction updateAction) { + JanusGraphManagement graphMgmt = graph.openManagement(); + CompletableFuture<IndexJobFuture>[] awaits = labels.stream() + .map(label -> + CompletableFuture.supplyAsync(() -> { + EdgeLabel relation = graphMgmt.getEdgeLabel(label); + RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label); + LOGGER.info("Updating relation index [{}] status from [{}] to [{}]", relation.name(), index.getIndexStatus(), updateAction); + return graphMgmt.updateIndex(index, updateAction); + })) + .toArray(CompletableFuture[]::new); + try { + CompletableFuture.allOf(awaits).get(); + LOGGER.info("Completed reindex actions"); + } catch (InterruptedException | ExecutionException e) { + LOGGER.error("Error while waiting for change in relation index status"); + e.printStackTrace(); + } graphMgmt.commit(); } - private static void makeEdgeLabels(JanusGraphManagement graphMgmt) { + /** + * Radical approach to avoiding index update failures. + * Indexes can get stuck in INSTALLED state, when there are stale transactions or JanusGraph instances. + * This is because a state change needs to be acknowledged by all instances before transitioning. + * @param graph + * @return + */ + private static void killTransactionsAndInstances(JanusGraph graph) { + graph.tx().rollback(); + final StandardJanusGraph janusGraph = (StandardJanusGraph) graph; + janusGraph.getOpenTransactions().stream().forEach(transaction -> { + LOGGER.debug("Closing open transaction [{}] before schema generation", transaction.toString()); + transaction.rollback(); + }); + + final JanusGraphManagement graphMgtForClosing = graph.openManagement(); + + Set<String> instances = graphMgtForClosing.getOpenInstances(); + LOGGER.info("Number of open instances: {}", instances.size()); + LOGGER.info("Currently open instances: [{}]", instances); + instances.stream() + // .filter(instance -> !instance.contains("graphadmin")) + .filter(instance -> !instance.contains("(current)")) + .forEach(instance -> { + LOGGER.debug("Closing open JanusGraph instance [{}] before reindexing procedure", instance); + graphMgtForClosing.forceCloseInstance(instance); + }); + graphMgtForClosing.commit(); + } + + private static void makeEdgeLabels(JanusGraphManagement graphMgmt, List<String> elementsToReindex, boolean dbNotEmpty) { try { EdgeIngestor edgeIngestor = SpringContextAware.getBean(EdgeIngestor.class); Set<String> labels = Optional.ofNullable(edgeIngestor.getAllCurrentRules()) - .map(collectValues(EdgeRule::getLabel)).orElseGet(HashSet::new); + .map(collectValues(EdgeRule::getLabel)).orElseGet(HashSet::new); labels.forEach(label -> { if (graphMgmt.containsRelationType(label)) { - LOGGER.debug(" EdgeLabel [{}] already existed. ", label); + LOGGER.debug(" EdgeLabel [{}] already exists.", label); } else { LOGGER.debug("Making EdgeLabel: [{}]", label); - graphMgmt.makeEdgeLabel(label).multiplicity(Multiplicity.valueOf("MULTI")).make(); + graphMgmt.makeEdgeLabel(label).multiplicity(Multiplicity.MULTI).make(); } + EdgeLabel relation = graphMgmt.getEdgeLabel(label); + RelationTypeIndex relationIndex = graphMgmt.getRelationIndex(relation, label); + if(relationIndex == null) { + LOGGER.debug("Creating edge index for relation: " + label); + graphMgmt.buildEdgeIndex(relation, label, Direction.BOTH, graphMgmt.getPropertyKey("aai-node-type")); + if(dbNotEmpty) { + LOGGER.info("DB not empty. Registering edge [{}] for later reindexing.", label); + elementsToReindex.add(relation.name()); + } + } else if(!relationIndex.getIndexStatus().equals(SchemaStatus.ENABLED)) { + LOGGER.info("Relation index was already created but is not in ENABLED status. Current status: [{}]", relationIndex.getIndexStatus()); + elementsToReindex.add(label); + } else { + LOGGER.debug("Edge index for label [{}] already exists", label); + }; }); } catch (EdgeRuleNotFoundException e) { LOGGER.error("Unable to find all rules {}", LogFormatTools.getStackTop(e)); diff --git a/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java index cb388ccc..d9994f52 100644 --- a/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java +++ b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java @@ -147,14 +147,14 @@ public class AAIGraph { private void loadSchema(JanusGraph graph) { // Load the propertyKeys, indexes and edge-Labels into the DB - JanusGraphManagement graphMgt = graph.openManagement(); - + boolean dbNotEmpty = graph.traversal().V().limit(1).hasNext(); logger.info("-- loading schema into JanusGraph"); if ("true".equals( - SpringContextAware.getApplicationContext().getEnvironment().getProperty("history.enabled", "false"))) { + SpringContextAware.getApplicationContext().getEnvironment().getProperty("history.enabled", "false"))) { + JanusGraphManagement graphMgt = graph.openManagement(); SchemaGenerator4Hist.loadSchemaIntoJanusGraph(graphMgt, IN_MEMORY); } else { - SchemaGenerator.loadSchemaIntoJanusGraph(graphMgt, IN_MEMORY); + SchemaGenerator.loadSchemaIntoJanusGraph(graph, IN_MEMORY, dbNotEmpty); } } diff --git a/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java b/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java index 51a2c221..cca11c6d 100644 --- a/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java +++ b/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java @@ -50,10 +50,9 @@ public class InMemoryGraph { Properties graphProps = new Properties(); graphProps.load(is); - JanusGraphManagement graphMgt = graph.openManagement(); if (builder.isSchemaEnabled) { LOGGER.info("Schema Enabled"); - SchemaGenerator.loadSchemaIntoJanusGraph(graphMgt, graphProps.getProperty("storage.backend")); + SchemaGenerator.loadSchemaIntoJanusGraph(graph, graphProps.getProperty("storage.backend"), false); } try (JanusGraphTransaction transaction = graph.newTransaction()) { LOGGER.info("Loading snapshot"); diff --git a/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java b/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java index ac08ae2c..a1206b59 100644 --- a/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java +++ b/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java @@ -47,7 +47,10 @@ import org.onap.aai.serialization.engines.TransactionalGraphEngine; import org.onap.aai.serialization.queryformats.exceptions.AAIFormatVertexException; import org.onap.aai.serialization.queryformats.utils.UrlBuilder; import org.onap.aai.setup.SchemaVersion; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.annotation.DirtiesContext.ClassMode; +@DirtiesContext(classMode=ClassMode.BEFORE_CLASS) public class ResourceWithSoTTest extends AAISetup { @Mock private UrlBuilder urlBuilder; @@ -78,7 +81,7 @@ public class ResourceWithSoTTest extends AAISetup { graph = TinkerGraph.open(); long currentTimeMs = System.currentTimeMillis(); - String timeNowInMs = Long.toString(currentTimeMs); + final String timeNowInMs = Long.toString(currentTimeMs); // PUT / CREATE jsonPutObj.addProperty("aai-created-ts", timeNowInMs); @@ -114,6 +117,24 @@ public class ResourceWithSoTTest extends AAISetup { createLoaderEngineSetup(); } + public void createLoaderEngineSetup() throws AAIException { + + loader = loaderFactory.createLoaderForVersion(factoryType, version); + + dbEngine = spy(new JanusGraphDBEngine(QueryStyle.TRAVERSAL, loader)); + serializer = new DBSerializer(version, dbEngine, factoryType, "Junit"); + resourceWithSoT = new ResourceWithSoT.Builder(loader, serializer, urlBuilder).build(); + + TransactionalGraphEngine.Admin spyAdmin = spy(dbEngine.asAdmin()); + + when(dbEngine.tx()).thenReturn(graph); + when(dbEngine.asAdmin()).thenReturn(spyAdmin); + + when(spyAdmin.getReadOnlyTraversalSource()) + .thenReturn(graph.traversal().withStrategies(ReadOnlyStrategy.instance())); + when(spyAdmin.getTraversalSource()).thenReturn(graph.traversal()); + } + // This test is to simulate a PUT request @Test public void testGetJsonFromVertexWithCreateVertex() throws AAIFormatVertexException, AAIException { @@ -148,24 +169,4 @@ public class ResourceWithSoTTest extends AAISetup { Optional<JsonObject> result = resourceWithSoT.getJsonFromVertex(null); assertFalse(result.isPresent()); } - - public void createLoaderEngineSetup() throws AAIException { - - if (loader == null) { - loader = loaderFactory.createLoaderForVersion(factoryType, version); - // loader = LoaderFactory.createLoaderForVersion(factoryType, version); - dbEngine = spy(new JanusGraphDBEngine(QueryStyle.TRAVERSAL, loader)); - serializer = new DBSerializer(version, dbEngine, factoryType, "Junit"); - resourceWithSoT = new ResourceWithSoT.Builder(loader, serializer, urlBuilder).build(); - - TransactionalGraphEngine.Admin spyAdmin = spy(dbEngine.asAdmin()); - - when(dbEngine.tx()).thenReturn(graph); - when(dbEngine.asAdmin()).thenReturn(spyAdmin); - - when(spyAdmin.getReadOnlyTraversalSource()) - .thenReturn(graph.traversal().withStrategies(ReadOnlyStrategy.instance())); - when(spyAdmin.getTraversalSource()).thenReturn(graph.traversal()); - } - } } diff --git a/aai-els-onap-logging/pom.xml b/aai-els-onap-logging/pom.xml index 820925ad..694abfef 100644 --- a/aai-els-onap-logging/pom.xml +++ b/aai-els-onap-logging/pom.xml @@ -4,7 +4,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-els-onap-logging</artifactId> diff --git a/aai-failover/pom.xml b/aai-failover/pom.xml index 32a3c49f..5df2733b 100644 --- a/aai-failover/pom.xml +++ b/aai-failover/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <modelVersion>4.0.0</modelVersion> diff --git a/aai-parent/pom.xml b/aai-parent/pom.xml index 72c907bf..2b5c191a 100644 --- a/aai-parent/pom.xml +++ b/aai-parent/pom.xml @@ -27,7 +27,7 @@ limitations under the License. <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-common</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> </parent> <artifactId>aai-parent</artifactId> <name>aai-parent</name> diff --git a/aai-rest/pom.xml b/aai-rest/pom.xml index 252c2837..cb9e6779 100644 --- a/aai-rest/pom.xml +++ b/aai-rest/pom.xml @@ -29,7 +29,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-rest</artifactId> diff --git a/aai-schema-abstraction/pom.xml b/aai-schema-abstraction/pom.xml index 24737ec5..e91a4934 100644 --- a/aai-schema-abstraction/pom.xml +++ b/aai-schema-abstraction/pom.xml @@ -28,7 +28,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> diff --git a/aai-schema-ingest/pom.xml b/aai-schema-ingest/pom.xml index 14706af8..3d530ae0 100644 --- a/aai-schema-ingest/pom.xml +++ b/aai-schema-ingest/pom.xml @@ -26,7 +26,7 @@ limitations under the License. <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-schema-ingest</artifactId> diff --git a/aai-utils/pom.xml b/aai-utils/pom.xml index e48e4c23..c83b8047 100644 --- a/aai-utils/pom.xml +++ b/aai-utils/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-parent</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <relativePath>../aai-parent/pom.xml</relativePath> </parent> <artifactId>aai-utils</artifactId> @@ -30,7 +30,7 @@ <groupId>org.onap.aai.aai-common</groupId> <artifactId>aai-common</artifactId> - <version>1.13.4-SNAPSHOT</version> + <version>1.13.5-SNAPSHOT</version> <packaging>pom</packaging> <name>aai-aai-common</name> <description>Contains all of the common code for resources and traversal repos</description> diff --git a/version.properties b/version.properties index c5bbf546..51a6e0a7 100644 --- a/version.properties +++ b/version.properties @@ -5,7 +5,7 @@ major_version=1 minor_version=13 -patch_version=4 +patch_version=5 base_version=${major_version}.${minor_version}.${patch_version} |