diff options
author | Fiete Ostkamp <Fiete.Ostkamp@telekom.de> | 2024-06-20 15:24:32 +0200 |
---|---|---|
committer | Fiete Ostkamp <Fiete.Ostkamp@telekom.de> | 2024-06-26 10:36:08 +0200 |
commit | bdcbdc723dfb3cd4c29fa3cdbe76ceb0df2a8033 (patch) | |
tree | 7ffe9ff48891be483d7524e84def321359b92e2b /aai-core/src/main | |
parent | c7e67a1cc3557db51f6510769b109ca347e862f0 (diff) |
Add gremlin-based pagination to aai-common
- enhance query building to support gremlin-based pagination
- pagination is supported in two variants: with and without the total count of elements [1]
- enhance query building to support gremlin-based sorting
- add query logging that is currently disabled
[1] due to the design of gremlin, including the total count results in a full graph scan.
As such there is the option to not include it, which should make it (much) faster for the first pages that are returned.
Issue-ID: AAI-3893
Change-Id: I6bc0c9b9f398556cc41a0a8f82e24e50c85e5690
Signed-off-by: Fiete Ostkamp <Fiete.Ostkamp@telekom.de>
Diffstat (limited to 'aai-core/src/main')
9 files changed, 335 insertions, 47 deletions
diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java index 24e5ec8b..8d7da688 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/GraphTraversalBuilder.java @@ -27,14 +27,18 @@ import com.google.common.collect.Multimap; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import org.apache.tinkerpop.gremlin.groovy.jsr223.GroovyTranslator; +import org.apache.tinkerpop.gremlin.process.traversal.Order; import org.apache.tinkerpop.gremlin.process.traversal.P; import org.apache.tinkerpop.gremlin.process.traversal.Path; import org.apache.tinkerpop.gremlin.process.traversal.Pop; +import org.apache.tinkerpop.gremlin.process.traversal.Scope; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.Traversal.Admin; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; @@ -53,15 +57,22 @@ import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Introspector; import org.onap.aai.introspection.Loader; +import org.onap.aai.query.entities.PaginationResult; import org.onap.aai.schema.enums.ObjectMetadata; import org.onap.aai.schema.enums.PropertyMetadata; import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The Class GraphTraversalBuilder. */ public abstract class GraphTraversalBuilder<E> extends QueryBuilder<E> { + private static final Logger LOGGER = LoggerFactory.getLogger(GraphTraversalBuilder.class); + + private final GroovyTranslator groovyTranslator = GroovyTranslator.of("source"); + protected GraphTraversal<Vertex, E> traversal = null; protected Admin<Vertex, E> completeTraversal = null; @@ -875,7 +886,7 @@ public abstract class GraphTraversalBuilder<E> extends QueryBuilder<E> { @Override public QueryBuilder<E> getContainerQuery() { - + if (this.parentStepIndex == 0) { return removeQueryStepsBetween(0, containerStepIndex); } else { @@ -933,6 +944,12 @@ public abstract class GraphTraversalBuilder<E> extends QueryBuilder<E> { if (start != null) { this.completeTraversal = traversal.asAdmin(); } else { + boolean queryLoggingEnabled = false; + if(queryLoggingEnabled) { + String query = groovyTranslator.translate(traversal.asAdmin().getBytecode()); + LOGGER.info("Query: {}", query); + } + admin = source.V().asAdmin(); TraversalHelper.insertTraversal(admin.getEndStep(), traversal.asAdmin(), admin); @@ -968,6 +985,79 @@ public abstract class GraphTraversalBuilder<E> extends QueryBuilder<E> { return this.completeTraversal.toList(); } + @Override + public QueryBuilder<E> sort(Sort sort) { + Order order = sort.getDirection() == Sort.Direction.ASC ? Order.asc : Order.desc; + traversal.order().by(sort.getProperty(), order); + stepIndex++; + return this; + } + + public PaginationResult<E> toPaginationResult(Pageable pageable) { + int page = pageable.getPage(); + int pageSize = pageable.getPageSize(); + if(pageable.isIncludeTotalCount()) { + return paginateWithTotalCount(page, pageSize); + } else { + return paginateWithoutTotalCount(page, pageSize); + } + } + + private PaginationResult<E> paginateWithoutTotalCount(int page, int pageSize) { + int startIndex = page * pageSize; + traversal.range(startIndex, startIndex + pageSize); + + if (this.completeTraversal == null) { + executeQuery(); + } + return new PaginationResult<E>(completeTraversal.toList()); + } + + private PaginationResult<E> paginateWithTotalCount(int page, int pageSize) { + int startIndex = page * pageSize; + traversal.fold().as("results","count") + .select("results","count"). + by(__.range(Scope.local, startIndex, startIndex + pageSize)). + by(__.count(Scope.local)); + + if (this.completeTraversal == null) { + executeQuery(); + } + return mapPaginationResult((Map<String,Object>) completeTraversal.next()); + } + + private PaginationResult<E> mapPaginationResult(Map<String,Object> result) { + Object objCount = result.get("count"); + Object vertices = result.get("results"); + if(vertices == null) { + return new PaginationResult<E>(Collections.emptyList() ,0L); + } + List<E> results = null; + if(vertices instanceof List) { + results = (List<E>) vertices; + } else if (vertices instanceof Vertex) { + results = Collections.singletonList((E) vertices); + } else { + String msg = String.format("Results must be a list or a vertex, but was %s", vertices.getClass().getName()); + LOGGER.error(msg); + throw new IllegalArgumentException(msg); + } + long totalCount = parseCount(objCount); + return new PaginationResult<E>(results, totalCount); + } + + private long parseCount(Object count) { + if(count instanceof String) { + return Long.parseLong((String) count); + } else if(count instanceof Integer) { + return Long.valueOf((int) count); + } else if (count instanceof Long) { + return (long) count; + } else { + throw new IllegalArgumentException("Count must be a string, integer, or long"); + } + } + protected QueryBuilder<Edge> has(String key, String value) { traversal.has(key, value); diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java index db1c78ae..292d88fd 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/GremlinQueryBuilder.java @@ -44,6 +44,7 @@ import org.onap.aai.edges.exceptions.EdgeRuleNotFoundException; import org.onap.aai.exceptions.AAIException; import org.onap.aai.introspection.Introspector; import org.onap.aai.introspection.Loader; +import org.onap.aai.query.entities.PaginationResult; import org.onap.aai.restcore.search.GremlinGroovyShell; import org.onap.aai.schema.enums.ObjectMetadata; import org.onap.aai.serialization.db.exceptions.NoEdgeRuleFoundException; @@ -956,8 +957,16 @@ public abstract class GremlinQueryBuilder<E> extends QueryBuilder<E> { return (QueryBuilder<Edge>) this; } - /* - * This is required for the subgraphstrategies to work - */ + @Override + public PaginationResult<E> toPaginationResult(Pageable pageable) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'toPaginationResult'"); + } + + @Override + public QueryBuilder<E> sort(Sort sort) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'sort'"); + } } diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/GremlinUnique.java b/aai-core/src/main/java/org/onap/aai/query/builder/GremlinUnique.java index 2b117c49..58495201 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/GremlinUnique.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/GremlinUnique.java @@ -35,6 +35,7 @@ import org.onap.aai.introspection.Loader; import org.onap.aai.parsers.query.QueryParser; import org.onap.aai.parsers.query.TraversalStrategy; import org.onap.aai.parsers.query.UniqueStrategy; +import org.onap.aai.query.entities.PaginationResult; /** * The Class GremlinUnique. diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/Pageable.java b/aai-core/src/main/java/org/onap/aai/query/builder/Pageable.java new file mode 100644 index 00000000..cb6cb62e --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/query/builder/Pageable.java @@ -0,0 +1,42 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2024 Deutsche Telekom. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.query.builder; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Object that contains the page and pageSize for pagination. + * `includeTotalCount` can optionally be provided to include the total count of objects in the response. + * Note that including the total count in the response will trigger a full graph scan (@see <a href="https://jayanta-mondal.medium.com/the-curious-case-of-pagination-for-gremlin-queries-d6fd9518620">The Curious Case of Pagination for Gremlin Queries</a>). + */ +@Getter +@RequiredArgsConstructor +public class Pageable { + private final int page; + private final int pageSize; + private boolean includeTotalCount = false; + + public Pageable includeTotalCount() { + this.includeTotalCount = true; + return this; + } +} diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java b/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java index 309ffa16..a22fc388 100644 --- a/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java +++ b/aai-core/src/main/java/org/onap/aai/query/builder/QueryBuilder.java @@ -47,6 +47,7 @@ import org.onap.aai.introspection.Introspector; import org.onap.aai.introspection.Loader; import org.onap.aai.parsers.query.QueryParser; import org.onap.aai.parsers.query.QueryParserStrategy; +import org.onap.aai.query.entities.PaginationResult; import org.springframework.context.ApplicationContext; /** @@ -501,7 +502,7 @@ public abstract class QueryBuilder<E> implements Iterator<E> { * This is necessary in cases such as "if the Optional Property 1 is sent, * find all Nodes of type A with edges to Nodes of type B with property 1, * otherwise, simply find all nodes of type A". - * + * * @param type * @param outNodeType * @param inNodeType @@ -520,7 +521,7 @@ public abstract class QueryBuilder<E> implements Iterator<E> { * This is necessary in cases such as "if the Optional Property 1 is sent, * find all Nodes of type A with edges to Nodes of type B with property 1, * otherwise, simply find all nodes of type A". - * + * * @param type * @param outNodeType * @param inNodeType @@ -744,6 +745,22 @@ public abstract class QueryBuilder<E> implements Iterator<E> { public abstract List<E> toList(); /** + * Paginate the resulting list. + * This is a final step that returns a PaginationResult. + * @param pageable object that contains page and page size + * @return returns a page of the results and the total count. + */ + public abstract PaginationResult<E> toPaginationResult(Pageable pageable); + + /** + * Sort the resulting list. + * @param sort object that contains the property to sort by and the direction + * @return returns the QueryBuilder for further query building + */ + public abstract QueryBuilder<E> sort(Sort sort); + + + /** * Used to skip step if there is an optional property missing. * * @param key diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/QueryOptions.java b/aai-core/src/main/java/org/onap/aai/query/builder/QueryOptions.java new file mode 100644 index 00000000..920bc284 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/query/builder/QueryOptions.java @@ -0,0 +1,41 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2024 Deutsche Telekom. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.query.builder; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +@Getter +@Builder +@AllArgsConstructor +public class QueryOptions { + @Builder.Default Pageable pageable = null; + @Builder.Default Sort sort = null; + + public QueryOptions(Pageable pageable) { + this.pageable = pageable; + } + + public QueryOptions(Sort sort) { + this.sort = sort; + } +} diff --git a/aai-core/src/main/java/org/onap/aai/query/builder/Sort.java b/aai-core/src/main/java/org/onap/aai/query/builder/Sort.java new file mode 100644 index 00000000..6009d610 --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/query/builder/Sort.java @@ -0,0 +1,37 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2024 Deutsche Telekom. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ + +package org.onap.aai.query.builder; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class Sort { + private final String property; + + @Builder.Default + Direction direction = Direction.ASC; + + public enum Direction { + ASC, DESC + } +} diff --git a/aai-core/src/main/java/org/onap/aai/query/entities/PaginationResult.java b/aai-core/src/main/java/org/onap/aai/query/entities/PaginationResult.java new file mode 100644 index 00000000..23e68ade --- /dev/null +++ b/aai-core/src/main/java/org/onap/aai/query/entities/PaginationResult.java @@ -0,0 +1,36 @@ +/** + * ============LICENSE_START======================================================= + * org.onap.aai + * ================================================================================ + * Copyright © 2024 Deutsche Telekom. All rights reserved. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============LICENSE_END========================================================= + */ +package org.onap.aai.query.entities; + +import java.util.List; + +import org.springframework.lang.Nullable; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@AllArgsConstructor +@RequiredArgsConstructor +public class PaginationResult <E> { + private final List<E> results; + @Nullable private Long totalCount = null; +} diff --git a/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java b/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java index 7ecdd6d5..7c652079 100644 --- a/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java +++ b/aai-core/src/main/java/org/onap/aai/rest/db/HttpEntry.java @@ -4,6 +4,8 @@ * ================================================================================ * Copyright © 2017-2018 AT&T Intellectual Property. All rights reserved. * ================================================================================ + * Modifications Copyright © 2024 Deutsche Telekom. + * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -50,6 +52,8 @@ import org.onap.aai.logging.ErrorLogHelper; import org.onap.aai.nodes.NodeIngestor; import org.onap.aai.parsers.query.QueryParser; import org.onap.aai.prevalidation.ValidationService; +import org.onap.aai.query.builder.QueryOptions; +import org.onap.aai.query.entities.PaginationResult; import org.onap.aai.rest.ueb.UEBNotification; import org.onap.aai.restcore.HttpMethod; import org.onap.aai.schema.enums.ObjectMetadata; @@ -219,16 +223,6 @@ public class HttpEntry { return dbEngine; } - public Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth, - Set<String> groups) throws AAIException { - return this.process(requests, sourceOfTruth, groups, true); - } - - public Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth) - throws AAIException { - return this.process(requests, sourceOfTruth, true); - } - /** * Checks the pagination bucket and pagination index variables to determine whether or not the user * requested paginated results @@ -310,22 +304,21 @@ public class HttpEntry { return this.totalVertices; } - /** - * Process. - * - * @param requests the requests - * @param sourceOfTruth the source of truth - * - * @return the pair - * @throws AAIException the AAI exception - */ - public Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth, - boolean enableResourceVersion) throws AAIException { - return this.process(requests, sourceOfTruth, Collections.EMPTY_SET, enableResourceVersion); + public Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth) throws AAIException { + return this.process(requests, sourceOfTruth, true); } - private Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth, - Set<String> groups, boolean enableResourceVersion) throws AAIException { + public Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth,boolean enableResourceVersion) throws AAIException { + return this.process(requests, sourceOfTruth, Collections.emptySet(), enableResourceVersion, null); + } + + public Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth, Set<String> groups) throws AAIException { + return this.process(requests, sourceOfTruth, groups, true, null); + } + + + public Pair<Boolean, List<Pair<URI, Response>>> process(List<DBRequest> requests, String sourceOfTruth, + Set<String> groups, boolean enableResourceVersion, QueryOptions queryOptions) throws AAIException { DBSerializer serializer = null; @@ -376,22 +369,21 @@ public class HttpEntry { uri = UriBuilder.fromPath(uriTemp).build(); boolean groupsAvailable = serializer.getGroups() != null && !serializer.getGroups().isEmpty(); - List<Vertex> queryResult = query.getQueryBuilder().toList(); - List<Vertex> vertices; - if (this.isPaginated()) { - List<Vertex> vertTemp = groupsAvailable ? queryResult.stream().filter((vx) -> { - return OwnerCheck.isAuthorized(groups, vx); - }).collect(Collectors.toList()) : queryResult; - this.setTotalsForPaging(vertTemp.size(), this.paginationBucket); - vertices = vertTemp.subList(((this.paginationIndex - 1) * this.paginationBucket), - Math.min((this.paginationBucket * this.paginationIndex), vertTemp.size())); + List<Vertex> queryResult; + PaginationResult<Vertex> paginationResult = null; + if(queryOptions != null && queryOptions.getPageable() != null) { + paginationResult = executePaginatedQuery(query, queryOptions); + queryResult = paginationResult.getResults(); } else { - vertices = groupsAvailable && queryResult.size() > 1 ? queryResult.stream().filter((vx) -> { - return OwnerCheck.isAuthorized(groups, vx); - }).collect(Collectors.toList()) : queryResult; - + queryResult = executeQuery(query, queryOptions); } + List<Vertex> vertices = groupsAvailable + ? queryResult.stream() + .filter(vertex -> OwnerCheck.isAuthorized(groups, vertex)) + .collect(Collectors.toList()) + : queryResult; + boolean isNewVertex; HttpHeaders headers = request.getHeaders(); outputMediaType = getMediaType(headers.getAcceptableMediaTypes()); @@ -686,10 +678,10 @@ public class HttpEntry { ) { String myvertid = v.id().toString(); - if (this.isPaginated()) { + if (paginationResult != null && paginationResult.getTotalCount() != null) { response = Response.status(status).header("vertex-id", myvertid) - .header("total-results", this.getTotalVertices()) - .header("total-pages", this.getTotalPaginationBuckets()).entity(result) + .header("total-results", paginationResult.getTotalCount()) + .entity(result) .type(outputMediaType).build(); } else { response = Response.status(status).header("vertex-id", myvertid).entity(result) @@ -749,6 +741,18 @@ public class HttpEntry { return Pair.with(success, responses); } + private List<Vertex> executeQuery(QueryParser query, QueryOptions queryOptions) { + return (queryOptions != null && queryOptions.getSort() != null) + ? query.getQueryBuilder().sort(queryOptions.getSort()).toList() + : query.getQueryBuilder().toList(); + } + + private PaginationResult<Vertex> executePaginatedQuery(QueryParser query, QueryOptions queryOptions) { + return queryOptions.getSort() != null + ? query.getQueryBuilder().sort(queryOptions.getSort()).toPaginationResult(queryOptions.getPageable()) + : query.getQueryBuilder().toPaginationResult(queryOptions.getPageable()); + } + /** * Generate notification events for the resulting db requests. */ @@ -815,7 +819,7 @@ public class HttpEntry { /** * Verifies that vertex has needed properties to generate on - * + * * @param vertex Vertex to be verified * @return <code>true</code> if vertex has necessary properties and exists */ @@ -1126,6 +1130,7 @@ public class HttpEntry { } } + @Deprecated public List<Object> getPaginatedVertexListForAggregateFormat(List<Object> aggregateVertexList) throws AAIException { List<Object> finalList = new Vector<>(); if (this.isPaginated()) { @@ -1154,6 +1159,16 @@ public class HttpEntry { return aggregateVertexList; } + /** + * This method is used to paginate the vertex list based on the pagination index and bucket size + * + * @deprecated + * This method is no longer supported. Use {@link #process(List, String, Set, boolean, QueryOptions)} instead. + * @param vertexList + * @return + * @throws AAIException + */ + @Deprecated public List<Object> getPaginatedVertexList(List<Object> vertexList) throws AAIException { List<Object> vertices; if (this.isPaginated()) { |