From d749a1072e23073e320ffe78c3f0576da0486279 Mon Sep 17 00:00:00 2001 From: Fiete Ostkamp Date: Tue, 23 Jan 2024 14:40:32 +0100 Subject: Update tinkerpop to 3.2.4 - work around hasStep inlining - add tests that more directly assert traversal state - add TinkerpopUpgradeTests test suite [1] - bump aai-common version to 1.13.2 [1] this allows to run the tests that are likely affected by a tinkerpop upgrade Issue-ID: AAI-3740 Change-Id: I86fb9b0c86802c7c0f74a5aae0976e81bf501eb0 Signed-off-by: Fiete Ostkamp --- .../java/org/onap/aai/parsers/uri/URIParser.java | 4 +- .../aai/query/builder/GraphTraversalBuilder.java | 27 +++++-- .../org/onap/aai/query/builder/QueryBuilder.java | 17 ++++- .../org/onap/aai/query/builder/TraversalQuery.java | 87 +++++++++++++++++++++- .../engines/TransactionalGraphEngine.java | 33 +++----- 5 files changed, 132 insertions(+), 36 deletions(-) (limited to 'aai-core/src/main') diff --git a/aai-core/src/main/java/org/onap/aai/parsers/uri/URIParser.java b/aai-core/src/main/java/org/onap/aai/parsers/uri/URIParser.java index 409631f5..0d25a15e 100644 --- a/aai-core/src/main/java/org/onap/aai/parsers/uri/URIParser.java +++ b/aai-core/src/main/java/org/onap/aai/parsers/uri/URIParser.java @@ -131,8 +131,8 @@ public class URIParser { uriKeys = queryParams; } else { for (String key : keys) { - part = UriUtils.decode(parts[i+1], "UTF-8"); - introspector.setValue(key, part); + String value = UriUtils.decode(parts[i+1], "UTF-8"); + introspector.setValue(key, value); // skip this for further processing i++; } diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java index 3c5c4489..c5c4512e 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java @@ -4,6 +4,8 @@ * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. * ================================================================================ + * * Modifications Copyright © 2024 DEUTSCHE TELEKOM AG. + * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -32,6 +34,7 @@ import java.util.Set; import org.apache.tinkerpop.gremlin.process.traversal.P; import org.apache.tinkerpop.gremlin.process.traversal.Path; +import org.apache.tinkerpop.gremlin.process.traversal.Step; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.Traversal.Admin; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; @@ -62,6 +65,9 @@ public abstract class GraphTraversalBuilder extends QueryBuilder { protected GraphTraversal traversal = null; protected Admin completeTraversal = null; + protected QueryBuilder containerQuery; + protected QueryBuilder parentQuery; + /** * Instantiates a new graph traversal builder. * @@ -73,6 +79,12 @@ public abstract class GraphTraversalBuilder extends QueryBuilder { } + public GraphTraversalBuilder(Loader loader, GraphTraversalSource source, GraphTraversal traversal) { + super(loader, source); + this.traversal = traversal; + + } + /** * Instantiates a new graph traversal builder. * @@ -336,11 +348,7 @@ public abstract class GraphTraversalBuilder extends QueryBuilder { } } - /** - * @{inheritDoc} - */ @Override - public QueryBuilder createContainerQuery(Introspector obj) { String type = obj.getChildDBName(); String abstractType = obj.getMetadata(ObjectMetadata.ABSTRACT); @@ -851,17 +859,18 @@ public abstract class GraphTraversalBuilder extends QueryBuilder { */ @Override public QueryBuilder getParentQuery() { - - return cloneQueryAtStep(parentStepIndex); + return this.parentQuery != null + ? this.parentQuery + : cloneQueryAtStep(parentStepIndex); } @Override public QueryBuilder getContainerQuery() { - + if (this.parentStepIndex == 0) { return removeQueryStepsBetween(0, containerStepIndex); } else { - return cloneQueryAtStep(containerStepIndex); + return this.containerQuery; } } @@ -870,11 +879,13 @@ public abstract class GraphTraversalBuilder extends QueryBuilder { */ @Override public void markParentBoundary() { + this.parentQuery = cloneQueryAtStep(stepIndex); parentStepIndex = stepIndex; } @Override public void markContainer() { + this.containerQuery = cloneQueryAtStep(stepIndex); containerStepIndex = stepIndex; } diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java index 2bc90726..66532529 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java @@ -4,6 +4,8 @@ * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. * ================================================================================ + * Modifications Copyright © 2024 DEUTSCHE TELEKOM AG. + * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -90,6 +92,11 @@ public abstract class QueryBuilder implements Iterator { this.loader = loader; } + /** + * Creates a new {@link QueryBuilder} that contains the traversal up to the provided index. + * @param index + * @return + */ protected abstract QueryBuilder cloneQueryAtStep(int index); /** @@ -266,9 +273,15 @@ public abstract class QueryBuilder implements Iterator { public abstract QueryBuilder createKeyQuery(Introspector obj); /** - * Creates the container query. + * Creates the container query.
+ * A container query is a query that will return a collection of objects: + * 
+     * /cloud-infrastructure/complexes/complex/key1
+     *           ↑              ↑        ↑
+     *       namespace      container  object
      *
-     * @param obj the obj
+     *
+ * @param obj the Introspector into the db schema * @return the query builder */ public abstract QueryBuilder createContainerQuery(Introspector obj); diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/TraversalQuery.java b/aai-core/src/main/java/org/onap/aai/query/builder/TraversalQuery.java index 579c3571..ca9d232c 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/TraversalQuery.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/TraversalQuery.java @@ -4,6 +4,8 @@ * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. * ================================================================================ + * Modifications Copyright © 2024 DEUTSCHE TELEKOM AG. + * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -23,12 +25,15 @@ package org.onap.aai.query.builder; import java.io.UnsupportedEncodingException; import java.net.URI; import java.util.List; +import java.util.stream.Collectors; import javax.ws.rs.core.MultivaluedMap; import org.apache.tinkerpop.gremlin.process.traversal.Step; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Introspector; @@ -52,6 +57,11 @@ public class TraversalQuery extends GraphTraversalBuilder { this.factory = new TraversalStrategy(this.loader, this); } + public TraversalQuery(Loader loader, GraphTraversalSource source, GraphTraversal traversal) { + super(loader, source, traversal); + this.factory = new TraversalStrategy(this.loader, this); + } + /** * Instantiates a new traversal query. * @@ -159,12 +169,85 @@ public class TraversalQuery extends GraphTraversalBuilder { @Override protected QueryBuilder removeQueryStepsBetween(int start, int end) { + if(start > 0) { + throw new IllegalArgumentException("A start index > 0 is currently not supported"); + } GraphTraversal clone = this.traversal.asAdmin().clone(); GraphTraversal.Admin cloneAdmin = clone.asAdmin(); - for (int i = end - 2; i >= start; i--) { - cloneAdmin.removeStep(i); + List steps = cloneAdmin.getSteps(); + + // TODO: Use containerAdjusted start index to support start > 0 + // TraversalQueryTest#removeQueryStepsBetweenTest27 can guide the implementation + int containerAdjusted = start > 0 + ? getContainerAdjustedStart(cloneAdmin, steps, start) + : start; + for (int i = start; i < end - 1; i++) { + Step step = steps.get(start); + if (step instanceof HasStep) { + List hasContainers = ((HasStep) step).getHasContainers(); + int hasContainerSize = hasContainers.size(); + boolean isEndWithinHasContainer = isEndWithinHasContainer(end, i, hasContainers); + if (isEndWithinHasContainer) { + int splitPosition = end - i - 1; + splitHasContainerAtPosition(cloneAdmin, hasContainers, start, splitPosition); + i += (hasContainerSize - splitPosition); + } else { + cloneAdmin.removeStep(start); + i += (hasContainerSize - 1); + } + } else { + cloneAdmin.removeStep(start); + } } return new TraversalQuery<>(cloneAdmin, loader, source, this); } + + private boolean isEndWithinHasContainer(int end, int i, List hasContainers) { + return (i + hasContainers.size()) >= end - 1; + } + + /** + * Since the has-step inlining that was introduced with tinkerpop version 3.2.4, + * a Step can be a HasContainer that can contain multiple steps. + * The start index needs to be adjusted to account for this fact + * @param cloneAdmin + * @param steps + * @param start + * @return + */ + private int getContainerAdjustedStart(GraphTraversal.Admin cloneAdmin, List steps, int start) { + int adjustedIndex = start; + for (int i = 0; i < start; i++) { + Step step = steps.get(i); + if (step instanceof HasStep) { + if(isEndWithinHasContainer(adjustedIndex, i, ((HasStep) step).getHasContainers())){ + adjustedIndex -= 1; + } + adjustedIndex -= ((HasStep) step).getHasContainers().size(); + } + } + return adjustedIndex; + } + + /** + * Split the hasContainer at the provided position and append everything + * after it to the step Array + * @param cloneAdmin + * @param hasContainers + * @param splitPosition + */ + private void splitHasContainerAtPosition(GraphTraversal.Admin cloneAdmin, + List hasContainers, int start, int splitPosition) { + List newContainers = hasContainers.stream() + .skip(splitPosition) + .collect(Collectors.toList()); + int replaceIndex = start; + cloneAdmin.removeStep(replaceIndex); + for (HasContainer hasContainer : newContainers) { + cloneAdmin.addStep(replaceIndex, new HasStep<>(cloneAdmin, hasContainer)); + replaceIndex++; + } + } + } diff --git a/aai-core/src/main/java/org/onap/aai/serialization/engines/TransactionalGraphEngine.java b/aai-core/src/main/java/org/onap/aai/serialization/engines/TransactionalGraphEngine.java index 73cdc3c8..54c8b7ce 100644 --- a/aai-core/src/main/java/org/onap/aai/serialization/engines/TransactionalGraphEngine.java +++ b/aai-core/src/main/java/org/onap/aai/serialization/engines/TransactionalGraphEngine.java @@ -4,6 +4,8 @@ * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. * ================================================================================ + * Modifications Copyright © 2024 DEUTSCHE TELEKOM AG. + * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -20,13 +22,13 @@ package org.onap.aai.serialization.engines; -import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.InlineFilterStrategy; -import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.LazyBarrierStrategy; import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategy; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Vertex; @@ -153,27 +155,14 @@ public abstract class TransactionalGraphEngine { public QueryBuilder getQueryBuilder(QueryStyle style, Loader loader) { GraphTraversalSource source = this.asAdmin().getTraversalSource(); - if (style.equals(QueryStyle.GREMLIN_TRAVERSAL)) { - return new GremlinTraversal<>(loader, source); - } else if (style.equals(QueryStyle.GREMLIN_UNIQUE)) { - return new GremlinUnique<>(loader, source); - } else if (style.equals(QueryStyle.GREMLINPIPELINE_TRAVERSAL)) { - // return new GremlinPipelineTraversal(loader); - } else if (style.equals(QueryStyle.TRAVERSAL)) { - return new TraversalQuery<>(loader, source != null ? source.withoutStrategies(InlineFilterStrategy.class) : source); - } else if (style.equals(QueryStyle.TRAVERSAL_URI)) { - return new TraversalURIOptimizedQuery<>(loader, source != null ? source.withoutStrategies(InlineFilterStrategy.class) : source); - } else if (style.equals(QueryStyle.HISTORY_TRAVERSAL)) { - throw new IllegalArgumentException("History Traversal needs history traversal source"); - } else if (style.equals(QueryStyle.HISTORY_GREMLIN_TRAVERSAL)) { - throw new IllegalArgumentException("History Gremlin Traversal needs history traversal source"); - } else { - throw new IllegalArgumentException("Query Builder type not recognized"); - } - return queryBuilder; + return this.getQueryBuilder(style, loader, source); } public QueryBuilder getQueryBuilder(QueryStyle style, Loader loader, GraphTraversalSource source) { + return this.getQueryBuilder(style, loader, source, (GraphTraversal) __.start()); + } + + public QueryBuilder getQueryBuilder(QueryStyle style, Loader loader, GraphTraversalSource source, GraphTraversal traversal) { if (style.equals(QueryStyle.GREMLIN_TRAVERSAL)) { return new GremlinTraversal<>(loader, source); } else if (style.equals(QueryStyle.GREMLIN_UNIQUE)) { @@ -181,9 +170,9 @@ public abstract class TransactionalGraphEngine { } else if (style.equals(QueryStyle.GREMLINPIPELINE_TRAVERSAL)) { // return new GremlinPipelineTraversal(loader); } else if (style.equals(QueryStyle.TRAVERSAL)) { - return new TraversalQuery<>(loader, source); + return new TraversalQuery<>(loader, source != null ? source.withoutStrategies(InlineFilterStrategy.class) : source, traversal); } else if (style.equals(QueryStyle.TRAVERSAL_URI)) { - return new TraversalURIOptimizedQuery<>(loader, source); + return new TraversalURIOptimizedQuery<>(loader, source != null ? source.withoutStrategies(InlineFilterStrategy.class) : source); } else if (style.equals(QueryStyle.HISTORY_TRAVERSAL)) { return new HistoryTraversalURIOptimizedQuery<>(loader, source); } else if (style.equals(QueryStyle.HISTORY_GREMLIN_TRAVERSAL)) { -- cgit 1.2.3-korg