Enhance SchemaGenerator to also generate indices for relationships

- generate edge indexes when they not already exist
- reindex existing indexes when they are not in status ENABLED
- try to recover indexes that are in INSTALLED state
- bump version to 1.13.5

Issue-ID: AAI-3782
Change-Id: I127e0f83b17485d7a6721206ade92cd50c89a51b
Signed-off-by: Fiete Ostkamp <Fiete.Ostkamp@telekom.de>

21 files changed, 378 insertions, 134 deletions

diff --git a/.gitignore b/.gitignore
index baf29672..413c5b3a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,4 +15,5 @@ bundleconfig-local/etc/logback.xml
 .vscode/
 .devcontainer/
 *.log
-aai-core/logs/
\ No newline at end of file
+aai-core/logs/
+.classpath
\ No newline at end of file
diff --git a/aai-aaf-auth/pom.xml b/aai-aaf-auth/pom.xml
index 9e69e603..12ca1b7b 100644
--- a/aai-aaf-auth/pom.xml
+++ b/aai-aaf-auth/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
     <artifactId>aai-aaf-auth</artifactId>
diff --git a/aai-annotations/pom.xml b/aai-annotations/pom.xml
index 60fa4eb3..e8ef6924 100644
--- a/aai-annotations/pom.xml
+++ b/aai-annotations/pom.xml
@@ -27,7 +27,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
     <artifactId>aai-annotations</artifactId>
diff --git a/aai-auth/pom.xml b/aai-auth/pom.xml
index 3922d6cf..ac251a42 100644
--- a/aai-auth/pom.xml
+++ b/aai-auth/pom.xml
@@ -27,7 +27,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
     <artifactId>aai-auth</artifactId>
diff --git a/aai-common-docker/aai-common-images/pom.xml b/aai-common-docker/aai-common-images/pom.xml
index 4cb93653..e15c6f2e 100644
--- a/aai-common-docker/aai-common-images/pom.xml
+++ b/aai-common-docker/aai-common-images/pom.xml
@@ -25,11 +25,11 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-common-docker</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
     </parent>
 
     <artifactId>aai-common-images</artifactId>
-    <version>1.13.4-SNAPSHOT</version>
+    <version>1.13.5-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>aai-aai-common-images</name>
     <description>Contains dockerfiles for aai-common images (alpine and ubuntu based).</description>
diff --git a/aai-common-docker/aai-haproxy-image/pom.xml b/aai-common-docker/aai-haproxy-image/pom.xml
index f77f4ef8..c2172309 100644
--- a/aai-common-docker/aai-haproxy-image/pom.xml
+++ b/aai-common-docker/aai-haproxy-image/pom.xml
@@ -25,11 +25,11 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-common-docker</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
     </parent>
 
     <artifactId>aai-haproxy-image</artifactId>
-    <version>1.13.4-SNAPSHOT</version>
+    <version>1.13.5-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>aai-aai-haproxy-image</name>
     <description>Contains dockerfiles for aai-haproxy image.</description>
diff --git a/aai-common-docker/pom.xml b/aai-common-docker/pom.xml
index e0e29fe2..80d7ece9 100644
--- a/aai-common-docker/pom.xml
+++ b/aai-common-docker/pom.xml
@@ -26,7 +26,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
 
diff --git a/aai-core/pom.xml b/aai-core/pom.xml
index 348d4eb0..fb646692 100644
--- a/aai-core/pom.xml
+++ b/aai-core/pom.xml
@@ -26,7 +26,7 @@ limitations under the License.
 	<parent>
 		<groupId>org.onap.aai.aai-common</groupId>
 		<artifactId>aai-parent</artifactId>
-		<version>1.13.4-SNAPSHOT</version>
+		<version>1.13.5-SNAPSHOT</version>
 		<relativePath>../aai-parent/pom.xml</relativePath>
 	</parent>
 	<artifactId>aai-core</artifactId>
diff --git a/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java b/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java
index 250c1263..6430df50 100644
--- a/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java
+++ b/aai-core/src/main/java/org/onap/aai/dbgen/SchemaGenerator.java
@@ -4,6 +4,8 @@
  * ================================================================================
  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
+ * Modifications Copyright © 2024 DEUTSCHE TELEKOM AG.
+ * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -22,21 +24,38 @@ package org.onap.aai.dbgen;
 
 import com.google.common.collect.Multimap;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
+import org.apache.tinkerpop.gremlin.structure.Direction;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.janusgraph.core.Cardinality;
+import org.janusgraph.core.EdgeLabel;
+import org.janusgraph.core.JanusGraph;
 import org.janusgraph.core.Multiplicity;
 import org.janusgraph.core.PropertyKey;
 import org.janusgraph.core.schema.ConsistencyModifier;
 import org.janusgraph.core.schema.JanusGraphIndex;
 import org.janusgraph.core.schema.JanusGraphManagement;
+import org.janusgraph.core.schema.JanusGraphManagement.IndexJobFuture;
+import org.janusgraph.core.schema.RelationTypeIndex;
+import org.janusgraph.core.schema.SchemaAction;
+import org.janusgraph.core.schema.SchemaStatus;
+import org.janusgraph.graphdb.database.StandardJanusGraph;
+import org.janusgraph.graphdb.database.management.ManagementSystem;
+import org.janusgraph.graphdb.database.management.RelationIndexStatusReport;
 import org.onap.aai.config.SpringContextAware;
 import org.onap.aai.edges.EdgeIngestor;
 import org.onap.aai.edges.EdgeRule;
@@ -53,6 +72,7 @@ import org.slf4j.LoggerFactory;
 public class SchemaGenerator {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(SchemaGenerator.class);
+    private static int indexRecoveryRetryCounter = 0;
 
     private SchemaGenerator() {
     }
@@ -63,7 +83,9 @@ public class SchemaGenerator {
      * @param graphMgmt
      *        the graph mgmt
      */
-    public static void loadSchemaIntoJanusGraph(final JanusGraphManagement graphMgmt, String backend) {
+    public static List<String> loadSchemaIntoJanusGraph(final JanusGraph graph, String backend, boolean dbNotEmpty) {
+        JanusGraphManagement graphMgmt = graph.openManagement();
+        final List<String> elementsToReindex = new ArrayList<>();
 
         try {
             AAIConfig.init();
@@ -85,121 +107,342 @@ public class SchemaGenerator {
         // multiplicty definitions depends on which two types of nodes are being
         // connected.
 
-        makeEdgeLabels(graphMgmt);
+        final Map<String, Introspector> objs = LoaderUtil.getLatestVersion().getAllObjects();
+        final Map<String, PropertyKey> seenProps = new HashMap<>();
+        
+        for (Introspector obj : objs.values()) {
+            createSchemaForObject(graphMgmt, seenProps, obj);
+        }
 
-        Map<String, Introspector> objs = LoaderUtil.getLatestVersion().getAllObjects();
-        Map<String, PropertyKey> seenProps = new HashMap<>();
+        makeEdgeLabels(graphMgmt, elementsToReindex, dbNotEmpty);
 
-        for (Introspector obj : objs.values()) {
-            for (String propName : obj.getProperties()) {
-                String dbPropName = propName;
-                Optional<String> alias = obj.getPropertyMetadata(propName, PropertyMetadata.DB_ALIAS);
-                if (alias.isPresent()) {
-                    dbPropName = alias.get();
+        LOGGER.info("-- About to call graphMgmt commit");
+        if (backend != null) {
+            LOGGER.info("Successfully loaded the schema to {}", backend);
+        }
+
+        graphMgmt.commit();
+        return elementsToReindex;
+    }
+
+    public static void reindexEdgeIndexes(final JanusGraph graph, Collection<String> edgeLabels) {
+        graph.tx().rollback();
+        if(edgeLabels.isEmpty()) {
+            LOGGER.info("Nothing to reindex.");
+            return;
+        }
+
+        ensureValidIndexState(graph, edgeLabels);
+
+        awaitRelationIndexStatus(graph, edgeLabels,SchemaStatus.REGISTERED);
+
+        LOGGER.info("Attempting to transition indexes in REGISTERED state to ENABLED");
+        updateRelationIndexes(graph, edgeLabels, SchemaAction.REINDEX);
+
+        ensureEnabledIndexState(graph, edgeLabels);
+    }
+
+    /**
+     * Indices need to be in status REGISTERED or ENABLED to allow reindexing.
+     * @param graph
+     * @param edgeLabels
+     */
+    private static void ensureEnabledIndexState(final JanusGraph graph, Collection<String> edgeLabels) {
+        JanusGraphManagement graphMgmt = graph.openManagement();
+        List<String> registeredIndexes = new ArrayList<>();
+        for(String label: edgeLabels) {
+            EdgeLabel relation = graphMgmt.getEdgeLabel(label);
+            RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label);
+            SchemaStatus indexStatus = index.getIndexStatus();
+            if(indexStatus.equals(SchemaStatus.REGISTERED)) {
+                LOGGER.info("Detected relation index [{}] that is not yet in ENABLED state", relation.name());
+                registeredIndexes.add(label);
+            }
+        }
+        graphMgmt.commit();
+
+        if(indexRecoveryRetryCounter <= 8 && !registeredIndexes.isEmpty()) {
+            indexRecoveryRetryCounter++;
+            LOGGER.info("[{}] indexes not yet in ENABLED state", registeredIndexes.size());
+            awaitRelationIndexStatus(graph, registeredIndexes,SchemaStatus.ENABLED);
+
+            ensureEnabledIndexState(graph, edgeLabels); // recursively call to make sure there is no invalid state
+        } else {
+            LOGGER.info("All indexes are in ENABLED state, exiting.");
+            return;
+        }
+    }
+
+    /**
+     * Indices need to be in status REGISTERED or ENABLED to allow reindexing.
+     * @param graph
+     * @param edgeLabels
+     */
+    private static void ensureValidIndexState(final JanusGraph graph, Collection<String> edgeLabels) {
+        JanusGraphManagement graphMgmt = graph.openManagement();
+        List<String> installedIndexes = new ArrayList<>();
+        List<String> disabledIndexes = new ArrayList<>();
+        for(String label: edgeLabels) {
+            EdgeLabel relation = graphMgmt.getEdgeLabel(label);
+            RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label);
+            SchemaStatus indexStatus = index.getIndexStatus();
+            if(indexStatus.equals(SchemaStatus.INSTALLED)) {
+                LOGGER.info("Detected relation index [{}] with invalid status [{}]", relation.name(), indexStatus);
+                installedIndexes.add(label);
+            } else if(indexStatus.equals(SchemaStatus.DISABLED)) {
+                LOGGER.info("Detected relation index [{}] with invalid status [{}]", relation.name(), indexStatus);
+                disabledIndexes.add(label);
+            }
+        }
+        graphMgmt.commit();
+
+        if(indexRecoveryRetryCounter <= 300 && (!installedIndexes.isEmpty() || !disabledIndexes.isEmpty())) {
+            indexRecoveryRetryCounter++;
+            if(!installedIndexes.isEmpty()) {
+                LOGGER.info("Attempting to transition indexes in INSTALLED state to REGISTERED");
+                updateRelationIndexes(graph, installedIndexes, SchemaAction.REGISTER_INDEX);
+                awaitRelationIndexStatus(graph, edgeLabels,SchemaStatus.REGISTERED);
+            }
+            if(!disabledIndexes.isEmpty()) {
+                LOGGER.info("Attempting to transition indexes in DISABLED state to ENABLED");
+                updateRelationIndexes(graph, disabledIndexes, SchemaAction.ENABLE_INDEX);
+                awaitRelationIndexStatus(graph, edgeLabels,SchemaStatus.ENABLED);
+            }
+            ensureValidIndexState(graph, edgeLabels); // recursively call to make sure there is no invalid state
+        } else {
+            return;
+        }
+    }
+
+    private static void createSchemaForObject(final JanusGraphManagement graphMgmt, Map<String, PropertyKey> seenProps,
+            Introspector obj) {
+        for (String propName : obj.getProperties()) {
+            String dbPropName = propName;
+            Optional<String> alias = obj.getPropertyMetadata(propName, PropertyMetadata.DB_ALIAS);
+            if (alias.isPresent()) {
+                dbPropName = alias.get();
+            }
+            if (graphMgmt.containsRelationType(dbPropName)) {
+                LOGGER.debug(" PropertyKey  [{}] already existed in the DB. ", dbPropName);
+            } else {
+                Class<?> type = obj.getClass(propName);
+                Cardinality cardinality = Cardinality.SINGLE;
+                boolean process = false;
+                if (obj.isListType(propName) && obj.isSimpleGenericType(propName)) {
+                    cardinality = Cardinality.SET;
+                    type = obj.getGenericTypeClass(propName);
+                    process = true;
+                } else if (obj.isSimpleType(propName)) {
+                    process = true;
                 }
-                if (graphMgmt.containsRelationType(dbPropName)) {
-                    LOGGER.debug(" PropertyKey  [{}] already existed in the DB. ", dbPropName);
-                } else {
-                    Class<?> type = obj.getClass(propName);
-                    Cardinality cardinality = Cardinality.SINGLE;
-                    boolean process = false;
-                    if (obj.isListType(propName) && obj.isSimpleGenericType(propName)) {
-                        cardinality = Cardinality.SET;
-                        type = obj.getGenericTypeClass(propName);
-                        process = true;
-                    } else if (obj.isSimpleType(propName)) {
-                        process = true;
-                    }
 
-                    if (process) {
-
-                        LOGGER.info("Creating PropertyKey: [{}], [{}], [{}]", dbPropName, type.getSimpleName(),
-                                cardinality);
-                        PropertyKey propK;
-                        if (!seenProps.containsKey(dbPropName)) {
-                            propK = graphMgmt.makePropertyKey(dbPropName).dataType(type).cardinality(cardinality)
-                                    .make();
-                            if (dbPropName.equals("aai-uri")) {
-                                String aai_uri_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false");
-                                LOGGER.info(" Info: aai_uri_lock_enabled:" + aai_uri_lock_enabled);
-                                if ("true".equals(aai_uri_lock_enabled)) {
-                                    LOGGER.info(" Lock is being set for aai-uri Property.");
-                                    graphMgmt.setConsistency(propK, ConsistencyModifier.LOCK);
-                                }
-                            } else if (dbPropName.equals("resource-version")) {
-                                String aai_rv_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false");
-                                LOGGER.info(" Info: aai_rv_lock_enabled:" + aai_rv_lock_enabled);
-                                if ("true".equals(aai_rv_lock_enabled)) {
-                                    LOGGER.info(" Lock is being set for resource-version Property.");
-                                    graphMgmt.setConsistency(propK, ConsistencyModifier.LOCK);
-                                }
+                if (process) {
+
+                    LOGGER.info("Creating PropertyKey: [{}], [{}], [{}]", dbPropName, type.getSimpleName(),
+                            cardinality);
+                    PropertyKey propertyKey;
+                    if (!seenProps.containsKey(dbPropName)) {
+                        propertyKey = graphMgmt.makePropertyKey(dbPropName).dataType(type).cardinality(cardinality)
+                                .make();
+                        if (dbPropName.equals("aai-uri")) {
+                            String aai_uri_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false");
+                            LOGGER.info(" Info: aai_uri_lock_enabled:" + aai_uri_lock_enabled);
+                            if ("true".equals(aai_uri_lock_enabled)) {
+                                LOGGER.info(" Lock is being set for aai-uri Property.");
+                                graphMgmt.setConsistency(propertyKey, ConsistencyModifier.LOCK);
+                            }
+                        } else if (dbPropName.equals("resource-version")) {
+                            String aai_rv_lock_enabled = AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false");
+                            LOGGER.info(" Info: aai_rv_lock_enabled:" + aai_rv_lock_enabled);
+                            if ("true".equals(aai_rv_lock_enabled)) {
+                                LOGGER.info(" Lock is being set for resource-version Property.");
+                                graphMgmt.setConsistency(propertyKey, ConsistencyModifier.LOCK);
                             }
-                            seenProps.put(dbPropName, propK);
-                        } else {
-                            propK = seenProps.get(dbPropName);
                         }
-                        if (graphMgmt.containsGraphIndex(dbPropName)) {
-                            LOGGER.debug(" Index  [{}] already existed in the DB. ", dbPropName);
+                        seenProps.put(dbPropName, propertyKey);
+                    } else {
+                        propertyKey = seenProps.get(dbPropName);
+                    }
+                    if (graphMgmt.containsGraphIndex(dbPropName)) {
+                        LOGGER.debug(" Index  [{}] already existed in the DB. ", dbPropName);
+                    } else {
+                        if (obj.getIndexedProperties().contains(propName)) {
+                            createIndexForProperty(graphMgmt, obj, propName, dbPropName, propertyKey);
                         } else {
-                            if (obj.getIndexedProperties().contains(propName)) {
-                                JanusGraphIndex indexG = null;
-                                if (obj.getUniqueProperties().contains(propName)) {
-                                    LOGGER.info("Add Unique index for PropertyKey: [{}]", dbPropName);
-                                    indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propK).unique()
-                                            .buildCompositeIndex();
-                                } else {
-                                    LOGGER.info("Add index for PropertyKey: [{}]", dbPropName);
-                                    indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propK)
-                                            .buildCompositeIndex();
-                                }
-                                if (indexG != null && dbPropName.equals("aai-uri")) {
-                                    String aai_uri_lock_enabled =
-                                            AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false");
-                                    LOGGER.info(" Info:: aai_uri_lock_enabled:" + aai_uri_lock_enabled);
-                                    if ("true".equals(aai_uri_lock_enabled)) {
-                                        LOGGER.info("Lock is being set for aai-uri Index.");
-                                        graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK);
-                                    }
-                                } else if (indexG != null && dbPropName.equals("resource-version")) {
-                                    String aai_rv_lock_enabled =
-                                            AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false");
-                                    LOGGER.info(" Info:: aai_rv_lock_enabled:" + aai_rv_lock_enabled);
-                                    if ("true".equals(aai_rv_lock_enabled)) {
-                                        LOGGER.info("Lock is being set for resource-version Index.");
-                                        graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK);
-                                    }
-                                }
-                            } else {
-                                LOGGER.info("No index added for PropertyKey: [{}]", dbPropName);
-                            }
+                            LOGGER.info("No index added for PropertyKey: [{}]", dbPropName);
                         }
                     }
                 }
             }
         }
+    }
 
-        LOGGER.info("-- About to call graphMgmt commit");
-        if (backend != null) {
-            LOGGER.info("Successfully loaded the schema to {}", backend);
+    private static void createIndexForProperty(final JanusGraphManagement graphMgmt, Introspector obj, String propName,
+            String dbPropName, PropertyKey propertyKey) {
+        JanusGraphIndex indexG = null;
+        if (obj.getUniqueProperties().contains(propName)) {
+            LOGGER.info("Add Unique index for PropertyKey: [{}]", dbPropName);
+            indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propertyKey).unique()
+                    .buildCompositeIndex();
+        } else {
+            LOGGER.info("Add index for PropertyKey: [{}]", dbPropName);
+            indexG = graphMgmt.buildIndex(dbPropName, Vertex.class).addKey(propertyKey)
+                    .buildCompositeIndex();
+        }
+        if (indexG != null && dbPropName.equals("aai-uri")) {
+            String aai_uri_lock_enabled =
+                    AAIConfig.get(AAIConstants.AAI_LOCK_URI_ENABLED, "false");
+            LOGGER.info(" Info:: aai_uri_lock_enabled:" + aai_uri_lock_enabled);
+            if ("true".equals(aai_uri_lock_enabled)) {
+                LOGGER.info("Lock is being set for aai-uri Index.");
+                graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK);
+            }
+        } else if (indexG != null && dbPropName.equals("resource-version")) {
+            String aai_rv_lock_enabled =
+                    AAIConfig.get(AAIConstants.AAI_LOCK_RV_ENABLED, "false");
+            LOGGER.info(" Info:: aai_rv_lock_enabled:" + aai_rv_lock_enabled);
+            if ("true".equals(aai_rv_lock_enabled)) {
+                LOGGER.info("Lock is being set for resource-version Index.");
+                graphMgmt.setConsistency(indexG, ConsistencyModifier.LOCK);
+            }
+        }
+    }
+
+    /**
+     * Debug method to print current index states.
+     * This can help diagnosing indexes that are stuck in INSTALLED state.
+     * @param graph
+     */
+    public static void printCurrentRelationIndexStates(JanusGraph graph) {
+        JanusGraphManagement graphMgmt = graph.openManagement();
+        EdgeIngestor edgeIngestor = SpringContextAware.getBean(EdgeIngestor.class);
+        try {
+            Set<String> edgeLabels = Optional.ofNullable(edgeIngestor.getAllCurrentRules())
+                        .map(collectValues(EdgeRule::getLabel)).orElseGet(HashSet::new);
+            edgeLabels.stream()
+                .forEach(label -> {
+                    EdgeLabel relation = graphMgmt.getEdgeLabel(label);
+                    RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label);
+                    LOGGER.info("Index state of relation index [{}] is [{}]",label, index.getIndexStatus());
+                });
+        } catch (EdgeRuleNotFoundException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * Jointly wait for a set of relation indexes to change status.
+     * Use this after calling {@link JanusGraphManagement#updateIndex(org.janusgraph.core.schema.Index, SchemaAction)}
+     * @param graph the JanusGraph
+     * @param labels the names of the indexes
+     * @param newStatus the new SchemaStatus
+     */
+    private static void awaitRelationIndexStatus(JanusGraph graph, Collection<String> labels, SchemaStatus newStatus) {
+        LOGGER.info("Awaiting index status [{}]", newStatus);;
+        CompletableFuture<RelationIndexStatusReport>[] awaits = labels.stream()
+            .map(label -> 
+                CompletableFuture.supplyAsync(() -> {
+                    try {
+                        return ManagementSystem
+                            .awaitRelationIndexStatus(graph, label, label)
+                            .status(newStatus)
+                            .call();
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                        return null;
+                    }
+                }))
+            .toArray(CompletableFuture[]::new);
+        try {
+            CompletableFuture.allOf(awaits).get();
+        } catch (InterruptedException | ExecutionException e) {
+            LOGGER.error("Error while waiting for change in relation index status");
+            e.printStackTrace();
         }
+        LOGGER.info("Completed waiting for index status [{}]", newStatus);
+    }
+
+    private static void updateRelationIndexes(JanusGraph graph, Collection<String> labels, SchemaAction updateAction) {
+        JanusGraphManagement graphMgmt = graph.openManagement();
 
+        CompletableFuture<IndexJobFuture>[] awaits = labels.stream()
+            .map(label -> 
+                CompletableFuture.supplyAsync(() -> {
+                    EdgeLabel relation = graphMgmt.getEdgeLabel(label);
+                    RelationTypeIndex index = graphMgmt.getRelationIndex(relation, label);
+                    LOGGER.info("Updating relation index [{}] status from [{}] to [{}]", relation.name(), index.getIndexStatus(), updateAction);
+                    return graphMgmt.updateIndex(index, updateAction);
+                }))
+            .toArray(CompletableFuture[]::new);
+        try {
+            CompletableFuture.allOf(awaits).get();
+            LOGGER.info("Completed reindex actions");
+        } catch (InterruptedException | ExecutionException e) {
+            LOGGER.error("Error while waiting for change in relation index status");
+            e.printStackTrace();
+        }
         graphMgmt.commit();
     }
 
-    private static void makeEdgeLabels(JanusGraphManagement graphMgmt) {
+    /**
+     * Radical approach to avoiding index update failures.
+     * Indexes can get stuck in INSTALLED state, when there are stale transactions or JanusGraph instances.
+     * This is because a state change needs to be acknowledged by all instances before transitioning.
+     * @param graph
+     * @return
+     */
+    private static void killTransactionsAndInstances(JanusGraph graph) {
+        graph.tx().rollback();
+        final StandardJanusGraph janusGraph = (StandardJanusGraph) graph;
+        janusGraph.getOpenTransactions().stream().forEach(transaction -> {
+                LOGGER.debug("Closing open transaction [{}] before schema generation", transaction.toString());
+                transaction.rollback();
+        });
+        
+        final JanusGraphManagement graphMgtForClosing = graph.openManagement();
+
+        Set<String> instances = graphMgtForClosing.getOpenInstances();
+        LOGGER.info("Number of open instances: {}", instances.size());
+        LOGGER.info("Currently open instances: [{}]", instances);
+        instances.stream()
+            // .filter(instance -> !instance.contains("graphadmin"))
+            .filter(instance -> !instance.contains("(current)"))
+            .forEach(instance -> {
+                    LOGGER.debug("Closing open JanusGraph instance [{}] before reindexing procedure", instance);
+                    graphMgtForClosing.forceCloseInstance(instance);
+            });
+        graphMgtForClosing.commit();
+    }
+
+    private static void makeEdgeLabels(JanusGraphManagement graphMgmt, List<String> elementsToReindex, boolean dbNotEmpty) {
         try {
             EdgeIngestor edgeIngestor = SpringContextAware.getBean(EdgeIngestor.class);
 
             Set<String> labels = Optional.ofNullable(edgeIngestor.getAllCurrentRules())
-                    .map(collectValues(EdgeRule::getLabel)).orElseGet(HashSet::new);
+            .map(collectValues(EdgeRule::getLabel)).orElseGet(HashSet::new);
 
             labels.forEach(label -> {
                 if (graphMgmt.containsRelationType(label)) {
-                    LOGGER.debug(" EdgeLabel  [{}] already existed. ", label);
+                    LOGGER.debug(" EdgeLabel  [{}] already exists.", label);
                 } else {
                     LOGGER.debug("Making EdgeLabel: [{}]", label);
-                    graphMgmt.makeEdgeLabel(label).multiplicity(Multiplicity.valueOf("MULTI")).make();
+                    graphMgmt.makeEdgeLabel(label).multiplicity(Multiplicity.MULTI).make();
                 }
+                EdgeLabel relation = graphMgmt.getEdgeLabel(label);
+                RelationTypeIndex relationIndex = graphMgmt.getRelationIndex(relation, label);
+                if(relationIndex == null) {
+                    LOGGER.debug("Creating edge index for relation: " + label);
+                    graphMgmt.buildEdgeIndex(relation, label, Direction.BOTH, graphMgmt.getPropertyKey("aai-node-type"));
+                    if(dbNotEmpty) {
+                        LOGGER.info("DB not empty. Registering edge [{}] for later reindexing.", label);
+                        elementsToReindex.add(relation.name());
+                    }
+                } else if(!relationIndex.getIndexStatus().equals(SchemaStatus.ENABLED)) {
+                    LOGGER.info("Relation index was already created but is not in ENABLED status. Current status: [{}]", relationIndex.getIndexStatus());
+                    elementsToReindex.add(label);
+                } else {
+                    LOGGER.debug("Edge index for label [{}] already exists", label);
+                };
             });
         } catch (EdgeRuleNotFoundException e) {
             LOGGER.error("Unable to find all rules {}", LogFormatTools.getStackTop(e));
diff --git a/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java
index cb388ccc..d9994f52 100644
--- a/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java
+++ b/aai-core/src/main/java/org/onap/aai/dbmap/AAIGraph.java
@@ -147,14 +147,14 @@ public class AAIGraph {
 
     private void loadSchema(JanusGraph graph) {
         // Load the propertyKeys, indexes and edge-Labels into the DB
-        JanusGraphManagement graphMgt = graph.openManagement();
-
+        boolean dbNotEmpty = graph.traversal().V().limit(1).hasNext();
         logger.info("-- loading schema into JanusGraph");
         if ("true".equals(
-                SpringContextAware.getApplicationContext().getEnvironment().getProperty("history.enabled", "false"))) {
+            SpringContextAware.getApplicationContext().getEnvironment().getProperty("history.enabled", "false"))) {
+            JanusGraphManagement graphMgt = graph.openManagement();
             SchemaGenerator4Hist.loadSchemaIntoJanusGraph(graphMgt, IN_MEMORY);
         } else {
-            SchemaGenerator.loadSchemaIntoJanusGraph(graphMgt, IN_MEMORY);
+            SchemaGenerator.loadSchemaIntoJanusGraph(graph, IN_MEMORY, dbNotEmpty);
         }
     }
 
diff --git a/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java b/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java
index 51a2c221..cca11c6d 100644
--- a/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java
+++ b/aai-core/src/main/java/org/onap/aai/dbmap/InMemoryGraph.java
@@ -50,10 +50,9 @@ public class InMemoryGraph {
 
             Properties graphProps = new Properties();
             graphProps.load(is);
-            JanusGraphManagement graphMgt = graph.openManagement();
             if (builder.isSchemaEnabled) {
                 LOGGER.info("Schema Enabled");
-                SchemaGenerator.loadSchemaIntoJanusGraph(graphMgt, graphProps.getProperty("storage.backend"));
+                SchemaGenerator.loadSchemaIntoJanusGraph(graph, graphProps.getProperty("storage.backend"), false);
             }
             try (JanusGraphTransaction transaction = graph.newTransaction()) {
                 LOGGER.info("Loading snapshot");
diff --git a/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java b/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java
index ac08ae2c..a1206b59 100644
--- a/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java
+++ b/aai-core/src/test/java/org/onap/aai/serialization/queryformats/ResourceWithSoTTest.java
@@ -47,7 +47,10 @@ import org.onap.aai.serialization.engines.TransactionalGraphEngine;
 import org.onap.aai.serialization.queryformats.exceptions.AAIFormatVertexException;
 import org.onap.aai.serialization.queryformats.utils.UrlBuilder;
 import org.onap.aai.setup.SchemaVersion;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.annotation.DirtiesContext.ClassMode;
 
+@DirtiesContext(classMode=ClassMode.BEFORE_CLASS)
 public class ResourceWithSoTTest extends AAISetup {
     @Mock
     private UrlBuilder urlBuilder;
@@ -78,7 +81,7 @@ public class ResourceWithSoTTest extends AAISetup {
         graph = TinkerGraph.open();
 
         long currentTimeMs = System.currentTimeMillis();
-        String timeNowInMs = Long.toString(currentTimeMs);
+        final String timeNowInMs = Long.toString(currentTimeMs);
 
         // PUT / CREATE
         jsonPutObj.addProperty("aai-created-ts", timeNowInMs);
@@ -114,6 +117,24 @@ public class ResourceWithSoTTest extends AAISetup {
         createLoaderEngineSetup();
     }
 
+    public void createLoaderEngineSetup() throws AAIException {
+
+        loader = loaderFactory.createLoaderForVersion(factoryType, version);
+
+        dbEngine = spy(new JanusGraphDBEngine(QueryStyle.TRAVERSAL, loader));
+        serializer = new DBSerializer(version, dbEngine, factoryType, "Junit");
+        resourceWithSoT = new ResourceWithSoT.Builder(loader, serializer, urlBuilder).build();
+
+        TransactionalGraphEngine.Admin spyAdmin = spy(dbEngine.asAdmin());
+
+        when(dbEngine.tx()).thenReturn(graph);
+        when(dbEngine.asAdmin()).thenReturn(spyAdmin);
+
+        when(spyAdmin.getReadOnlyTraversalSource())
+                .thenReturn(graph.traversal().withStrategies(ReadOnlyStrategy.instance()));
+        when(spyAdmin.getTraversalSource()).thenReturn(graph.traversal());
+    }
+
     // This test is to simulate a PUT request
     @Test
     public void testGetJsonFromVertexWithCreateVertex() throws AAIFormatVertexException, AAIException {
@@ -148,24 +169,4 @@ public class ResourceWithSoTTest extends AAISetup {
         Optional<JsonObject> result = resourceWithSoT.getJsonFromVertex(null);
         assertFalse(result.isPresent());
     }
-
-    public void createLoaderEngineSetup() throws AAIException {
-
-        if (loader == null) {
-            loader = loaderFactory.createLoaderForVersion(factoryType, version);
-            // loader = LoaderFactory.createLoaderForVersion(factoryType, version);
-            dbEngine = spy(new JanusGraphDBEngine(QueryStyle.TRAVERSAL, loader));
-            serializer = new DBSerializer(version, dbEngine, factoryType, "Junit");
-            resourceWithSoT = new ResourceWithSoT.Builder(loader, serializer, urlBuilder).build();
-
-            TransactionalGraphEngine.Admin spyAdmin = spy(dbEngine.asAdmin());
-
-            when(dbEngine.tx()).thenReturn(graph);
-            when(dbEngine.asAdmin()).thenReturn(spyAdmin);
-
-            when(spyAdmin.getReadOnlyTraversalSource())
-                    .thenReturn(graph.traversal().withStrategies(ReadOnlyStrategy.instance()));
-            when(spyAdmin.getTraversalSource()).thenReturn(graph.traversal());
-        }
-    }
 }
diff --git a/aai-els-onap-logging/pom.xml b/aai-els-onap-logging/pom.xml
index 820925ad..694abfef 100644
--- a/aai-els-onap-logging/pom.xml
+++ b/aai-els-onap-logging/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
     <artifactId>aai-els-onap-logging</artifactId>
diff --git a/aai-failover/pom.xml b/aai-failover/pom.xml
index 32a3c49f..5df2733b 100644
--- a/aai-failover/pom.xml
+++ b/aai-failover/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
diff --git a/aai-parent/pom.xml b/aai-parent/pom.xml
index 72c907bf..2b5c191a 100644
--- a/aai-parent/pom.xml
+++ b/aai-parent/pom.xml
@@ -27,7 +27,7 @@ limitations under the License.
   <parent>
     <groupId>org.onap.aai.aai-common</groupId>
     <artifactId>aai-common</artifactId>
-    <version>1.13.4-SNAPSHOT</version>
+    <version>1.13.5-SNAPSHOT</version>
   </parent>
   <artifactId>aai-parent</artifactId>
   <name>aai-parent</name>
diff --git a/aai-rest/pom.xml b/aai-rest/pom.xml
index 252c2837..cb9e6779 100644
--- a/aai-rest/pom.xml
+++ b/aai-rest/pom.xml
@@ -29,7 +29,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
     <artifactId>aai-rest</artifactId>
diff --git a/aai-schema-abstraction/pom.xml b/aai-schema-abstraction/pom.xml
index 24737ec5..e91a4934 100644
--- a/aai-schema-abstraction/pom.xml
+++ b/aai-schema-abstraction/pom.xml
@@ -28,7 +28,7 @@
   <parent>
     <groupId>org.onap.aai.aai-common</groupId>
     <artifactId>aai-parent</artifactId>
-    <version>1.13.4-SNAPSHOT</version>
+    <version>1.13.5-SNAPSHOT</version>
     <relativePath>../aai-parent/pom.xml</relativePath>
   </parent>
 
diff --git a/aai-schema-ingest/pom.xml b/aai-schema-ingest/pom.xml
index 14706af8..3d530ae0 100644
--- a/aai-schema-ingest/pom.xml
+++ b/aai-schema-ingest/pom.xml
@@ -26,7 +26,7 @@ limitations under the License.
 	<parent>
 		<groupId>org.onap.aai.aai-common</groupId>
 		<artifactId>aai-parent</artifactId>
-		<version>1.13.4-SNAPSHOT</version>
+		<version>1.13.5-SNAPSHOT</version>
 		<relativePath>../aai-parent/pom.xml</relativePath>
 	</parent>
 	<artifactId>aai-schema-ingest</artifactId>
diff --git a/aai-utils/pom.xml b/aai-utils/pom.xml
index e48e4c23..c83b8047 100644
--- a/aai-utils/pom.xml
+++ b/aai-utils/pom.xml
@@ -27,7 +27,7 @@
     <parent>
         <groupId>org.onap.aai.aai-common</groupId>
         <artifactId>aai-parent</artifactId>
-        <version>1.13.4-SNAPSHOT</version>
+        <version>1.13.5-SNAPSHOT</version>
         <relativePath>../aai-parent/pom.xml</relativePath>
     </parent>
     <artifactId>aai-utils</artifactId>
diff --git a/pom.xml b/pom.xml
index fa17a6d0..08b2c124 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@
 
     <groupId>org.onap.aai.aai-common</groupId>
     <artifactId>aai-common</artifactId>
-    <version>1.13.4-SNAPSHOT</version>
+    <version>1.13.5-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>aai-aai-common</name>
     <description>Contains all of the common code for resources and traversal repos</description>
diff --git a/version.properties b/version.properties
index c5bbf546..51a6e0a7 100644
--- a/version.properties
+++ b/version.properties
@@ -5,7 +5,7 @@
 
 major_version=1
 minor_version=13
-patch_version=4
+patch_version=5
 
 base_version=${major_version}.${minor_version}.${patch_version}