Pivoting a TraversalURIOptimisedQuery is broken in release 1.13.2

- fix query building for TraversalURIOptimizedQuery
- enable DataLinkTests for TRAVERSAL_URI again
- add further tests to test pivoting more directly
- roll back version to 1.13.2 to rerelease fixed version
- add python six dependency to fix upstream issue with sphinxcontrib-swagger dependency

Issue-ID: AAI-3758
Signed-off-by: Fiete Ostkamp <Fiete.Ostkamp@telekom.de>
Change-Id: I3e54216e00d0491d64265e149f55cf8aba6fabd9

5 files changed, 180 insertions, 44 deletions

diff --git a/aai-core/pom.xml b/aai-core/pom.xml
index 682aac7d..e5d3d718 100644
--- a/aai-core/pom.xml
+++ b/aai-core/pom.xml
@@ -26,7 +26,7 @@ limitations under the License.
 	<parent>
 		<groupId>org.onap.aai.aai-common</groupId>
 		<artifactId>aai-parent</artifactId>
-		<version>1.13.3-SNAPSHOT</version>
+		<version>1.13.2-SNAPSHOT</version>
 		<relativePath>../aai-parent/pom.xml</relativePath>
 	</parent>
 	<artifactId>aai-core</artifactId>
diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/TraversalURIOptimizedQuery.java b/aai-core/src/main/java/org/onap/aai/query/builder/TraversalURIOptimizedQuery.java
index 0e2a9cad..b96847c2 100644
--- a/aai-core/src/main/java/org/onap/aai/query/builder/TraversalURIOptimizedQuery.java
+++ b/aai-core/src/main/java/org/onap/aai/query/builder/TraversalURIOptimizedQuery.java
@@ -4,6 +4,8 @@
  * ================================================================================
  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
+ * Modifications Copyright © 2024 Deutsche Telekom.
+ * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -20,16 +22,20 @@
 
 package org.onap.aai.query.builder;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.tinkerpop.gremlin.process.traversal.Step;
 import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer;
 import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.onap.aai.db.props.AAIProperties;
@@ -37,7 +43,7 @@ import org.onap.aai.introspection.Introspector;
 import org.onap.aai.introspection.Loader;
 import org.onap.aai.schema.enums.ObjectMetadata;
 
-public class TraversalURIOptimizedQuery<E> extends TraversalQuery {
+public class TraversalURIOptimizedQuery<E> extends TraversalQuery<E> {
 
     protected Map<Integer, String> stepToAaiUri = new HashMap<>();
 
@@ -46,21 +52,27 @@ public class TraversalURIOptimizedQuery<E> extends TraversalQuery {
         optimize = true;
     }
 
+    public TraversalURIOptimizedQuery(Loader loader, GraphTraversalSource source,
+            GraphTraversal<Vertex, Vertex> traversal) {
+        super(loader, source);
+        optimize = true;
+    }
+
     public TraversalURIOptimizedQuery(Loader loader, GraphTraversalSource source, Vertex start) {
         super(loader, source, start);
         optimize = true;
     }
 
     protected TraversalURIOptimizedQuery(GraphTraversal traversal, Loader loader, GraphTraversalSource source,
-            GraphTraversalBuilder gtb) {
-        super(traversal, loader, source, gtb);
+            GraphTraversalBuilder graphTraversalBuilder) {
+        super(traversal, loader, source, graphTraversalBuilder);
         optimize = true;
     }
 
     protected TraversalURIOptimizedQuery(GraphTraversal traversal, Loader loader, GraphTraversalSource source,
-            GraphTraversalBuilder gtb, Map<Integer, String> stepToAaiUri) {
-        super(traversal, loader, source, gtb);
-        optimize = gtb.optimize;
+            GraphTraversalBuilder graphTraversalBuilder, Map<Integer, String> stepToAaiUri) {
+        super(traversal, loader, source, graphTraversalBuilder);
+        optimize = graphTraversalBuilder.optimize;
         this.stepToAaiUri = stepToAaiUri;
     }
 
@@ -74,7 +86,7 @@ public class TraversalURIOptimizedQuery<E> extends TraversalQuery {
         }
 
         if (start == null) {
-            Traversal.Admin admin = source.V().asAdmin();
+            Traversal.Admin<Vertex, Vertex> admin = source.V().asAdmin();
             TraversalHelper.insertTraversal(admin.getEndStep(), completeTraversal, admin);
 
             this.completeTraversal = (Traversal.Admin<Vertex, E>) admin;
@@ -84,31 +96,74 @@ public class TraversalURIOptimizedQuery<E> extends TraversalQuery {
     }
 
     private Traversal.Admin<Vertex, E> pivotTraversal(Traversal.Admin<Vertex, E> traversalAdmin) {
-
-        List<Step> steps = traversalAdmin.getSteps();
-
-        Traversal.Admin<Vertex, E> traversalAdminStart = traversalAdmin.clone();
-
         // if we do not have an index or other conditions do no optimization
         if (stepToAaiUri.isEmpty()) {
             return traversalAdmin;
         }
 
-        int lastURIStepKey = getLastURIStepKey();
+        Traversal.Admin<Vertex, E> traversalAdminStart = traversalAdmin.clone();
 
+        List<Step> steps = traversalAdmin.getSteps();
         // clean up traversal steps
         for (int i = 0; i < steps.size(); i++) {
             traversalAdminStart.removeStep(0);
         }
 
-        ((GraphTraversal<Vertex, E>) traversalAdminStart).has(AAIProperties.AAI_URI, stepToAaiUri.get(lastURIStepKey));
-        for (int i = lastURIStepKey; i < steps.size(); i++) {
+        int lastURIStepIndex = getLastURIStepIndex();
+        ((GraphTraversal<Vertex, E>) traversalAdminStart).has(AAIProperties.AAI_URI,
+                stepToAaiUri.get(lastURIStepIndex));
+
+        ImmutablePair<Integer, Integer> indexAndStepCountTuple = getHasContainerAdjustedIndexAndSplitPosition(steps,
+                lastURIStepIndex);
+        int adjustedIndex = indexAndStepCountTuple.getKey();
+        for (int i = adjustedIndex; i < steps.size(); i++) {
+            Step step = steps.get(i);
+            boolean isFirstStep = i == adjustedIndex;
+            if (isFirstStep && step instanceof HasStep) {
+                int splitPosition = indexAndStepCountTuple.getValue();
+                List<HasContainer> newContainers = ((HasStep<?>) step).getHasContainers().stream()
+                        .skip(splitPosition)
+                        .collect(Collectors.toList());
+                traversalAdminStart
+                        .addStep(new HasStep<Vertex>(traversalAdminStart, newContainers.toArray(new HasContainer[0])));
+                i++;
+            }
             traversalAdminStart.addStep(steps.get(i));
         }
 
         return traversalAdminStart;
     }
 
+    /**
+     * Adjust lastURIStepIndex by the number of steps that are in hasContainers.
+     * A HasContainer can contain multiple steps, which skews the original index.
+     * Returns the step index and split position inside the hasContainer
+     * 
+     * @param steps            the list of steps to go through
+     * @param lastURIStepIndex the list index to adjust
+     * @return a Tuple<Integer, Integer> of the form (index, splitPosition)
+     */
+    private ImmutablePair<Integer, Integer> getHasContainerAdjustedIndexAndSplitPosition(List<Step> steps,
+            int lastURIStepIndex) {
+        int stepCount = 0;
+        for (int j = 0; j <= lastURIStepIndex; j++) {
+            Step step = steps.get(j);
+            if (step instanceof HasStep) {
+                stepCount += ((HasStep<?>) step).getHasContainers().size();
+            } else {
+                stepCount++;
+            }
+            if (stepCount == lastURIStepIndex) {
+                int splitPosition = stepCount + 1 - lastURIStepIndex;
+                return new ImmutablePair<>(j + 1, splitPosition);
+            } else if (stepCount > lastURIStepIndex) {
+                int splitPosition = stepCount + 1 - lastURIStepIndex;
+                return new ImmutablePair<>(j, splitPosition);
+            }
+        }
+        return new ImmutablePair<>(lastURIStepIndex, lastURIStepIndex);
+    }
+
     @Override
     public QueryBuilder<Vertex> createKeyQuery(Introspector obj) {
         super.createKeyQuery(obj);
@@ -159,14 +214,14 @@ public class TraversalURIOptimizedQuery<E> extends TraversalQuery {
         }
 
         if (!stepToAaiUri.isEmpty()) {
-            uri = stepToAaiUri.get(getLastURIStepKey()) + uri;
+            uri = stepToAaiUri.get(getLastURIStepIndex()) + uri;
         }
 
         return Optional.of(uri);
     }
 
-    protected int getLastURIStepKey() {
-        return stepToAaiUri.keySet().stream().mapToInt(Integer::intValue).max().getAsInt();
+    protected int getLastURIStepIndex() {
+        return Collections.max(stepToAaiUri.keySet());
     }
 
     private Map<Integer, String> getStepToAaiUriWithoutStepGreaterThan(final int index) {
diff --git a/aai-core/src/test/java/org/onap/aai/TinkerpopUpgradeTests.java b/aai-core/src/test/java/org/onap/aai/TinkerpopUpgradeTests.java
index 81c3ed81..799f6060 100644
--- a/aai-core/src/test/java/org/onap/aai/TinkerpopUpgradeTests.java
+++ b/aai-core/src/test/java/org/onap/aai/TinkerpopUpgradeTests.java
@@ -27,17 +27,25 @@ import org.junit.runners.Suite.SuiteClasses;
 import org.onap.aai.introspection.sideeffect.DataLinkTest;
 import org.onap.aai.parsers.query.GraphTraversalTest;
 import org.onap.aai.query.builder.TraversalQueryTest;
+import org.onap.aai.query.builder.TraversalURIOptimizedQueryTest;
 
 /**
- * Collection of tests that are likely affected by a Tinkerpop update
- * This is to more targedly run these tests during development:
+ * Collection of tests that are likely affected by a Tinkerpop update.
+ * This is done to more directly run these tests during development:
+ * 
+ * <pre>
+ * <code>
  * mvn test -Dcheckstyle.skip -Dtest=TinkerpopUpgradeTests
+ * </code>
+ * </pre>
  */
 @RunWith(Categories.class)
 @IncludeCategory(TinkerpopUpgrade.class)
-@SuiteClasses({ 
-  DataLinkTest.class,
-  GraphTraversalTest.class,
-  TraversalQueryTest.class
+@SuiteClasses({
+    DataLinkTest.class,
+    GraphTraversalTest.class,
+    TraversalQueryTest.class,
+    TraversalURIOptimizedQueryTest.class
 })
-public class TinkerpopUpgradeTests {}
+public class TinkerpopUpgradeTests {
+}
diff --git a/aai-core/src/test/java/org/onap/aai/introspection/sideeffect/DataLinkTest.java b/aai-core/src/test/java/org/onap/aai/introspection/sideeffect/DataLinkTest.java
index 7ccf1305..138723aa 100644
--- a/aai-core/src/test/java/org/onap/aai/introspection/sideeffect/DataLinkTest.java
+++ b/aai-core/src/test/java/org/onap/aai/introspection/sideeffect/DataLinkTest.java
@@ -4,6 +4,8 @@
  * ================================================================================
  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
+ * Modifications Copyright © 2024 Deutsche Telekom.
+ * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -63,10 +65,8 @@ import org.onap.aai.introspection.Introspector;
 import org.onap.aai.introspection.Loader;
 import org.onap.aai.introspection.LoaderUtil;
 import org.onap.aai.introspection.ModelType;
-import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
 import org.onap.aai.parsers.query.QueryParser;
 import org.onap.aai.query.builder.QueryBuilder;
-import org.onap.aai.query.builder.TraversalQuery;
 import org.onap.aai.restcore.util.URITools;
 import org.onap.aai.serialization.db.DBSerializer;
 import org.onap.aai.serialization.engines.JanusGraphDBEngine;
@@ -95,7 +95,7 @@ public class DataLinkTest extends DataLinkSetup {
 
     @Parameterized.Parameters(name = "QueryStyle.{0}")
     public static Collection<Object[]> data() {
-        return Arrays.asList(new Object[][] {{QueryStyle.TRAVERSAL}});
+        return Arrays.asList(new Object[][] {{QueryStyle.TRAVERSAL},{QueryStyle.TRAVERSAL_URI}});
     }
 
     @BeforeClass
@@ -231,23 +231,9 @@ public class DataLinkTest extends DataLinkSetup {
         GraphTraversal<Vertex, Vertex> traversal = __.<Vertex>start();
         
         QueryParser uriQuery = dbEngine.getQueryBuilder(this.queryStyle, loader, source, traversal).createQueryFromURI(uri, map);
-        // assertEquals(6, traversal.asAdmin().getSteps().size());
-        // assertEquals("HasStep([vpn-id.eq(modifyKey)])", traversal.asAdmin().getSteps().get(0).toString());
-        // assertEquals("HasStep([aai-node-type.eq(vpn-binding)])", traversal.asAdmin().getSteps().get(1).toString());
-        // assertEquals("VertexStep(IN,[org.onap.relationships.inventory.BelongsTo],vertex)", traversal.asAdmin().getSteps().get(2).toString());
-        // assertEquals("HasStep([aai-node-type.eq(route-target)])", traversal.asAdmin().getSteps().get(3).toString());
-        // assertEquals("HasStep([global-route-target.eq(modifyTargetKey2)])", traversal.asAdmin().getSteps().get(4).toString());
-        // assertEquals("HasStep([route-target-role.eq(modifyRoleKey2)])", traversal.asAdmin().getSteps().get(5).toString());
         List<Vertex> results = uriQuery.getQueryBuilder().toList();
 
         assertEquals(0, results.size());
-        // assertEquals(traversal.asAdmin().getSteps().size(), 6);
-        // assertEquals("HasStep([vpn-id.eq(modifyKey)])", traversal.asAdmin().getSteps().get(0).toString());
-        // assertEquals("HasStep([aai-node-type.eq(vpn-binding)])", traversal.asAdmin().getSteps().get(1).toString());
-        // assertEquals("VertexStep(IN,[org.onap.relationships.inventory.BelongsTo],vertex)", traversal.asAdmin().getSteps().get(2).toString());
-        // assertEquals("HasStep([aai-node-type.eq(route-target)])", traversal.asAdmin().getSteps().get(3).toString());
-        // assertEquals("HasStep([global-route-target.eq(modifyTargetKey2)])", traversal.asAdmin().getSteps().get(4).toString());
-        // assertEquals("HasStep([route-target-role.eq(modifyRoleKey2)])", traversal.asAdmin().getSteps().get(5).toString());
 
         QueryBuilder<Vertex> queryBuilder = uriQuery.getQueryBuilder().getContainerQuery()
         .getVerticesByProperty(AAIProperties.LINKED, true);
@@ -403,4 +389,5 @@ public class DataLinkTest extends DataLinkSetup {
         g.tx().rollback();
 
     }
-}
+ }
+ 
\ No newline at end of file
diff --git a/aai-core/src/test/java/org/onap/aai/query/builder/TraversalURIOptimizedQueryTest.java b/aai-core/src/test/java/org/onap/aai/query/builder/TraversalURIOptimizedQueryTest.java
index 7ed90711..afd73de6 100644
--- a/aai-core/src/test/java/org/onap/aai/query/builder/TraversalURIOptimizedQueryTest.java
+++ b/aai-core/src/test/java/org/onap/aai/query/builder/TraversalURIOptimizedQueryTest.java
@@ -4,6 +4,8 @@
  * ================================================================================
  * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved.
  * ================================================================================
+ * Modifications Copyright © 2024 Deutsche Telekom.
+ * ================================================================================
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -20,11 +22,32 @@
 
 package org.onap.aai.query.builder;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.tinkerpop.gremlin.process.traversal.Path;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
 import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree;
 import org.apache.tinkerpop.gremlin.structure.Edge;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.JanusGraph;
+import org.janusgraph.core.JanusGraphFactory;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.onap.aai.TinkerpopUpgrade;
+import org.onap.aai.db.props.AAIProperties;
+import org.onap.aai.introspection.Loader;
+import org.onap.aai.introspection.LoaderUtil;
+import org.onap.aai.introspection.exceptions.AAIUnknownObjectException;
 
+@Category(TinkerpopUpgrade.class)
 public class TraversalURIOptimizedQueryTest extends TraversalQueryTest {
 
     @Override
@@ -71,4 +94,67 @@ public class TraversalURIOptimizedQueryTest extends TraversalQueryTest {
     protected QueryBuilder<Path> getNewPathTraversalWithTestEdgeRules() {
         return new TraversalURIOptimizedQuery<>(loader, g);
     }
+
+    @Test
+    public void thatTraversalIsPivotedWithinHasContainer() throws AAIUnknownObjectException {
+        JanusGraph graph = JanusGraphFactory.build().set("storage.backend", "inmemory").open();
+        GraphTraversalSource source = graph.newTransaction().traversal();
+        final Loader loader = LoaderUtil.getLatestVersion();
+        GraphTraversal<Vertex, Vertex> traversal = (GraphTraversal<Vertex, Vertex>) __.<Vertex>start();
+        GraphTraversalBuilder graphTraversalBuilder = new TraversalURIOptimizedQuery<>(loader, source);
+        Map<Integer, String> stepToAaiUri = Collections.singletonMap(6, "/smth");
+        TraversalURIOptimizedQuery traversalQuery = new TraversalURIOptimizedQuery<>(traversal, loader, source,
+                graphTraversalBuilder, stepToAaiUri);
+        traversalQuery.limit(1);
+        traversalQuery.has("propertyKey", "value");
+        traversalQuery.has("propertyKey2", "value2");
+        traversalQuery.limit(2);
+        traversalQuery.has("propertyKey3", "value3");
+        traversalQuery.has("propertyKey4", "value4");
+        traversalQuery.has("propertyKey5", "value5");
+        traversalQuery.limit(3);
+        traversalQuery.limit(4);
+
+        traversalQuery.executeQuery();
+        String query = traversalQuery.completeTraversal.getSteps().toString();
+        assertEquals(
+                "[GraphStep(vertex,[]), HasStep([aai-uri.eq(/smth)]), HasStep([propertyKey5.eq(value5)]), RangeGlobalStep(0,3), RangeGlobalStep(0,4)]",
+                query);
+    }
+
+    @Test
+    public void thatTraversalIsPivottedAtRegularStep() throws AAIUnknownObjectException {
+        JanusGraph graph = JanusGraphFactory.build().set("storage.backend", "inmemory").open();
+        GraphTraversalSource source = graph.newTransaction().traversal();
+        final Loader loader = LoaderUtil.getLatestVersion();
+        GraphTraversal<Vertex, Vertex> traversal = (GraphTraversal<Vertex, Vertex>) __.<Vertex>start();
+        GraphTraversalBuilder graphTraversalBuilder = new TraversalURIOptimizedQuery<>(loader, source);
+        Map<Integer, String> stepToAaiUri = Collections.singletonMap(3, "/smth");
+        TraversalURIOptimizedQuery traversalQuery = new TraversalURIOptimizedQuery<>(traversal, loader, source,
+                graphTraversalBuilder, stepToAaiUri);
+        traversalQuery.limit(1);
+        traversalQuery.has("propertyKey", "value");
+        traversalQuery.has("propertyKey2", "value2");
+        traversalQuery.limit(2);
+        traversalQuery.has("propertyKey3", "value3");
+        traversalQuery.has("propertyKey4", "value4");
+        traversalQuery.has("propertyKey5", "value5");
+        traversalQuery.limit(3);
+        traversalQuery.limit(4);
+
+        GraphTraversal<Vertex, Vertex> expected = __.<Vertex>start()
+                .has(AAIProperties.AAI_URI, "/smth")
+                .limit(2)
+                .has("propertyKey3", "value3")
+                .has("propertyKey4", "value4")
+                .has("propertyKey5", "value5")
+                .limit(3)
+                .limit(4);
+        List<Step> expectedSteps = expected.asAdmin().getSteps();
+        traversalQuery.executeQuery();
+        List<Step> rawActual = traversalQuery.completeTraversal.getSteps();
+        List<Step> actualStep = rawActual.subList(1, rawActual.size()); // remove GraphStep since I found no way to add
+                                                                        // it to the expected traversal
+        assertArrayEquals(expectedSteps.toArray(), actualStep.toArray());
+    }
 }