From f248b5d9b794d5bdff59145406e0398d6fdcafa4 Mon Sep 17 00:00:00 2001 From: "rajesh.kumar" Date: Tue, 25 Apr 2023 11:58:35 +0530 Subject: Support pagination in query across all anchors(ep4) Add pagination query parameters in query across all anchors API pagination parameters (pageIndex and pageSize) are optional default is to query all fragments each pageSize represents number of records(number of anchors) TotalRecords is returned in response header to find number of pages. - If pagination option is provided in request then query number of anchors equal to pageSize. pageIndex is used for setting offset. - return number of records(one anchor per record) as per pagesize and pageSize Issue-ID: CPS-1605 Change-ID: I73f97f986a817d423f93a8d922dcd9647b2504bc Signed-off-by: rajesh.kumar --- cps-ri/pom.xml | 2 +- .../spi/impl/CpsDataPersistenceServiceImpl.java | 72 +++++++++++++++++----- .../onap/cps/spi/impl/utils/CpsValidatorImpl.java | 13 ++++ .../cps/spi/repository/FragmentQueryBuilder.java | 57 +++++++++++++++-- .../cps/spi/repository/FragmentRepository.java | 5 ++ .../repository/FragmentRepositoryCpsPathQuery.java | 8 ++- .../FragmentRepositoryCpsPathQueryImpl.java | 15 ++++- .../cps/spi/impl/utils/CpsValidatorSpec.groovy | 10 +++ 8 files changed, 160 insertions(+), 22 deletions(-) (limited to 'cps-ri') diff --git a/cps-ri/pom.xml b/cps-ri/pom.xml index 6207393740..941d44734a 100644 --- a/cps-ri/pom.xml +++ b/cps-ri/pom.xml @@ -33,7 +33,7 @@ cps-ri - 0.30 + 0.28 diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java index f4afe3d73b..19302d67aa 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java @@ -23,7 +23,8 @@ package org.onap.cps.spi.impl; -import com.google.common.base.Strings; +import static org.onap.cps.spi.PaginationOption.NO_PAGINATION; + import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet.Builder; import io.micrometer.core.annotation.Timed; @@ -48,6 +49,7 @@ import org.onap.cps.cpspath.parser.CpsPathUtil; import org.onap.cps.cpspath.parser.PathParsingException; import org.onap.cps.spi.CpsDataPersistenceService; import org.onap.cps.spi.FetchDescendantsOption; +import org.onap.cps.spi.PaginationOption; import org.onap.cps.spi.entities.AnchorEntity; import org.onap.cps.spi.entities.DataspaceEntity; import org.onap.cps.spi.entities.FragmentEntity; @@ -79,8 +81,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService private final SessionManager sessionManager; private static final String REG_EX_FOR_OPTIONAL_LIST_INDEX = "(\\[@.+?])?)"; - private static final String QUERY_ACROSS_ANCHORS = null; - private static final AnchorEntity ALL_ANCHORS = null; @Override public void addChildDataNodes(final String dataspaceName, final String anchorName, @@ -288,8 +288,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService public List queryDataNodes(final String dataspaceName, final String anchorName, final String cpsPath, final FetchDescendantsOption fetchDescendantsOption) { final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); - final AnchorEntity anchorEntity = Strings.isNullOrEmpty(anchorName) ? ALL_ANCHORS - : anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName); + final AnchorEntity anchorEntity = anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName); final CpsPathQuery cpsPathQuery; try { cpsPathQuery = CpsPathUtil.getCpsPathQuery(cpsPath); @@ -298,28 +297,60 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService } Collection fragmentEntities; - if (anchorEntity == ALL_ANCHORS) { - fragmentEntities = fragmentRepository.findByDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery); + fragmentEntities = fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery); + if (cpsPathQuery.hasAncestorAxis()) { + final Collection ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery); + fragmentEntities = fragmentRepository.findByAnchorAndXpathIn(anchorEntity, ancestorXpaths); + } + fragmentEntities = fragmentRepository.prefetchDescendantsOfFragmentEntities(fetchDescendantsOption, + fragmentEntities); + return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); + } + + @Override + @Timed(value = "cps.data.persistence.service.datanode.query.anchors", + description = "Time taken to query data nodes across all anchors or list of anchors") + public List queryDataNodesAcrossAnchors(final String dataspaceName, final String cpsPath, + final FetchDescendantsOption fetchDescendantsOption, + final PaginationOption paginationOption) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final CpsPathQuery cpsPathQuery; + try { + cpsPathQuery = CpsPathUtil.getCpsPathQuery(cpsPath); + } catch (final PathParsingException e) { + throw new CpsPathException(e.getMessage()); + } + + final List anchorIds; + if (paginationOption == NO_PAGINATION) { + anchorIds = Collections.EMPTY_LIST; } else { - fragmentEntities = fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery); + anchorIds = getAnchorIdsForPagination(dataspaceEntity, cpsPathQuery, paginationOption); + if (anchorIds.isEmpty()) { + return Collections.emptyList(); + } } + Collection fragmentEntities = + fragmentRepository.findByDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery, anchorIds); + if (cpsPathQuery.hasAncestorAxis()) { final Collection ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery); - if (anchorEntity == ALL_ANCHORS) { + if (anchorIds.isEmpty()) { fragmentEntities = fragmentRepository.findByDataspaceAndXpathIn(dataspaceEntity, ancestorXpaths); } else { - fragmentEntities = fragmentRepository.findByAnchorAndXpathIn(anchorEntity, ancestorXpaths); + fragmentEntities = fragmentRepository.findByAnchorIdsAndXpathIn( + anchorIds.toArray(new Long[0]), ancestorXpaths.toArray(new String[0])); } + } fragmentEntities = fragmentRepository.prefetchDescendantsOfFragmentEntities(fetchDescendantsOption, fragmentEntities); return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); } - @Override - public List queryDataNodesAcrossAnchors(final String dataspaceName, final String cpsPath, - final FetchDescendantsOption fetchDescendantsOption) { - return queryDataNodes(dataspaceName, QUERY_ACROSS_ANCHORS, cpsPath, fetchDescendantsOption); + private List getAnchorIdsForPagination(final DataspaceEntity dataspaceEntity, final CpsPathQuery cpsPathQuery, + final PaginationOption paginationOption) { + return fragmentRepository.findAnchorIdsForPagination(dataspaceEntity, cpsPathQuery, paginationOption); } private List createDataNodesFromFragmentEntities(final FetchDescendantsOption fetchDescendantsOption, @@ -358,6 +389,19 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService sessionManager.lockAnchor(sessionId, dataspaceName, anchorName, timeoutInMilliseconds); } + @Override + public Integer countAnchorsForDataspaceAndCpsPath(final String dataspaceName, final String cpsPath) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final CpsPathQuery cpsPathQuery; + try { + cpsPathQuery = CpsPathUtil.getCpsPathQuery(cpsPath); + } catch (final PathParsingException e) { + throw new CpsPathException(e.getMessage()); + } + final List anchorIdList = getAnchorIdsForPagination(dataspaceEntity, cpsPathQuery, NO_PAGINATION); + return anchorIdList.size(); + } + private static Set processAncestorXpath(final Collection fragmentEntities, final CpsPathQuery cpsPathQuery) { final Set ancestorXpath = new HashSet<>(); diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java index 1f61ee3c51..c727388b25 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Collection; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.onap.cps.spi.PaginationOption; import org.onap.cps.spi.exceptions.DataValidationException; import org.onap.cps.spi.utils.CpsValidator; import org.springframework.stereotype.Component; @@ -54,4 +55,16 @@ public class CpsValidatorImpl implements CpsValidator { } } } + + @Override + public void validatePaginationOption(final PaginationOption paginationOption) { + if (PaginationOption.NO_PAGINATION == paginationOption) { + return; + } + + if (!paginationOption.isValidPaginationOption()) { + throw new DataValidationException("Pagination validation error.", + "Invalid page index or size"); + } + } } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java index e371035ba5..0c43d62bc7 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java @@ -21,8 +21,10 @@ package org.onap.cps.spi.repository; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Queue; import javax.persistence.EntityManager; @@ -32,6 +34,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.cpspath.parser.CpsPathPrefixType; import org.onap.cps.cpspath.parser.CpsPathQuery; +import org.onap.cps.spi.PaginationOption; import org.onap.cps.spi.entities.AnchorEntity; import org.onap.cps.spi.entities.DataspaceEntity; import org.onap.cps.spi.entities.FragmentEntity; @@ -56,7 +59,8 @@ public class FragmentQueryBuilder { * @return a executable query object */ public Query getQueryForAnchorAndCpsPath(final AnchorEntity anchorEntity, final CpsPathQuery cpsPathQuery) { - return getQueryForDataspaceOrAnchorAndCpsPath(anchorEntity.getDataspace(), anchorEntity, cpsPathQuery); + return getQueryForDataspaceOrAnchorAndCpsPath(anchorEntity.getDataspace(), + anchorEntity, cpsPathQuery, Collections.EMPTY_LIST); } /** @@ -67,13 +71,45 @@ public class FragmentQueryBuilder { * @return a executable query object */ public Query getQueryForDataspaceAndCpsPath(final DataspaceEntity dataspaceEntity, - final CpsPathQuery cpsPathQuery) { - return getQueryForDataspaceOrAnchorAndCpsPath(dataspaceEntity, ACROSS_ALL_ANCHORS, cpsPathQuery); + final CpsPathQuery cpsPathQuery, + final List anchorIdsForPagination) { + return getQueryForDataspaceOrAnchorAndCpsPath(dataspaceEntity, ACROSS_ALL_ANCHORS, + cpsPathQuery, anchorIdsForPagination); + } + + /** + * Get query for dataspace, cps path, page index and page size. + * @param dataspaceEntity data space entity + * @param cpsPathQuery cps path query + * @param paginationOption pagination option + * @return query for given dataspace, cps path and pagination parameters + */ + public Query getQueryForAnchorIdsForPagination(final DataspaceEntity dataspaceEntity, + final CpsPathQuery cpsPathQuery, + final PaginationOption paginationOption) { + final StringBuilder sqlStringBuilder = new StringBuilder(); + final Map queryParameters = new HashMap<>(); + sqlStringBuilder.append("SELECT distinct(fragment.anchor_id) FROM fragment " + + "JOIN anchor ON anchor.id = fragment.anchor_id WHERE dataspace_id = :dataspaceId"); + queryParameters.put("dataspaceId", dataspaceEntity.getId()); + addXpathSearch(cpsPathQuery, sqlStringBuilder, queryParameters); + addLeafConditions(cpsPathQuery, sqlStringBuilder); + addTextFunctionCondition(cpsPathQuery, sqlStringBuilder, queryParameters); + addContainsFunctionCondition(cpsPathQuery, sqlStringBuilder, queryParameters); + if (PaginationOption.NO_PAGINATION != paginationOption) { + sqlStringBuilder.append(" ORDER BY fragment.anchor_id"); + addPaginationCondition(sqlStringBuilder, queryParameters, paginationOption); + } + + final Query query = entityManager.createNativeQuery(sqlStringBuilder.toString()); + setQueryParameters(query, queryParameters); + return query; } private Query getQueryForDataspaceOrAnchorAndCpsPath(final DataspaceEntity dataspaceEntity, final AnchorEntity anchorEntity, - final CpsPathQuery cpsPathQuery) { + final CpsPathQuery cpsPathQuery, + final List anchorIdsForPagination) { final StringBuilder sqlStringBuilder = new StringBuilder(); final Map queryParameters = new HashMap<>(); @@ -81,6 +117,10 @@ public class FragmentQueryBuilder { sqlStringBuilder.append("SELECT fragment.* FROM fragment JOIN anchor ON anchor.id = fragment.anchor_id" + " WHERE dataspace_id = :dataspaceId"); queryParameters.put("dataspaceId", dataspaceEntity.getId()); + if (!anchorIdsForPagination.isEmpty()) { + sqlStringBuilder.append(" AND anchor_id IN (:anchorIdsForPagination)"); + queryParameters.put("anchorIdsForPagination", anchorIdsForPagination); + } } else { sqlStringBuilder.append("SELECT * FROM fragment WHERE anchor_id = :anchorId"); queryParameters.put("anchorId", anchorEntity.getId()); @@ -107,6 +147,15 @@ public class FragmentQueryBuilder { } } + private static void addPaginationCondition(final StringBuilder sqlStringBuilder, + final Map queryParameters, + final PaginationOption paginationOption) { + final Integer offset = (paginationOption.getPageIndex() - 1) * paginationOption.getPageSize(); + sqlStringBuilder.append(" LIMIT :pageSize OFFSET :offset"); + queryParameters.put("pageSize", paginationOption.getPageSize()); + queryParameters.put("offset", offset); + } + private static Integer getTextValueAsInt(final CpsPathQuery cpsPathQuery) { try { return Integer.parseInt(cpsPathQuery.getTextFunctionConditionValue()); diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java index 303af5bc47..11b2b07733 100755 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java @@ -68,6 +68,11 @@ public interface FragmentRepository extends JpaRepository, return findByDataspaceIdAndXpathIn(dataspaceEntity.getId(), xpaths.toArray(new String[0])); } + @Query(value = "SELECT * FROM fragment WHERE anchor_id IN (:anchorIds)" + + " AND xpath = ANY (:xpaths)", nativeQuery = true) + List findByAnchorIdsAndXpathIn(@Param("anchorIds") Long[] anchorIds, + @Param("xpaths") String[] xpaths); + @Query(value = "SELECT * FROM fragment WHERE anchor_id = :anchorId LIMIT 1", nativeQuery = true) Optional findOneByAnchorId(@Param("anchorId") long anchorId); diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java index de0c060148..9c279618b0 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java @@ -23,6 +23,7 @@ package org.onap.cps.spi.repository; import java.util.List; import org.onap.cps.cpspath.parser.CpsPathQuery; +import org.onap.cps.spi.PaginationOption; import org.onap.cps.spi.entities.AnchorEntity; import org.onap.cps.spi.entities.DataspaceEntity; import org.onap.cps.spi.entities.FragmentEntity; @@ -30,5 +31,10 @@ import org.onap.cps.spi.entities.FragmentEntity; public interface FragmentRepositoryCpsPathQuery { List findByAnchorAndCpsPath(AnchorEntity anchorEntity, CpsPathQuery cpsPathQuery); - List findByDataspaceAndCpsPath(DataspaceEntity dataspaceEntity, CpsPathQuery cpsPathQuery); + List findByDataspaceAndCpsPath(DataspaceEntity dataspaceEntity, + CpsPathQuery cpsPathQuery, List anchorIds); + + List findAnchorIdsForPagination(DataspaceEntity dataspaceEntity, CpsPathQuery cpsPathQuery, + PaginationOption paginationOption); + } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java index 6cc6495b8d..1ba9d1a4c5 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java @@ -29,6 +29,7 @@ import javax.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.cpspath.parser.CpsPathQuery; +import org.onap.cps.spi.PaginationOption; import org.onap.cps.spi.entities.AnchorEntity; import org.onap.cps.spi.entities.DataspaceEntity; import org.onap.cps.spi.entities.FragmentEntity; @@ -55,11 +56,21 @@ public class FragmentRepositoryCpsPathQueryImpl implements FragmentRepositoryCps @Override @Transactional public List findByDataspaceAndCpsPath(final DataspaceEntity dataspaceEntity, - final CpsPathQuery cpsPathQuery) { - final Query query = fragmentQueryBuilder.getQueryForDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery); + final CpsPathQuery cpsPathQuery, final List anchorIds) { + final Query query = fragmentQueryBuilder.getQueryForDataspaceAndCpsPath( + dataspaceEntity, cpsPathQuery, anchorIds); final List fragmentEntities = query.getResultList(); log.debug("Fetched {} fragment entities by cps path across all anchors.", fragmentEntities.size()); return fragmentEntities; } + @Override + @Transactional + public List findAnchorIdsForPagination(final DataspaceEntity dataspaceEntity, final CpsPathQuery cpsPathQuery, + final PaginationOption paginationOption) { + final Query query = fragmentQueryBuilder.getQueryForAnchorIdsForPagination( + dataspaceEntity, cpsPathQuery, paginationOption); + return query.getResultList(); + } + } diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy index 345089c931..8d348443c7 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy @@ -20,6 +20,7 @@ package org.onap.cps.spi.impl.utils +import org.onap.cps.spi.PaginationOption import org.onap.cps.spi.exceptions.DataValidationException import spock.lang.Specification @@ -64,4 +65,13 @@ class CpsValidatorSpec extends Specification { then: 'a data validation exception is thrown' thrown(DataValidationException) } + + def 'Validate Pagination option with invalid page index and size.'() { + when: 'the pagination option is validated using invalid options' + objectUnderTest.validatePaginationOption(new PaginationOption(-5, -2)) + then: 'a data validation exception is thrown' + def exceptionThrown = thrown(DataValidationException) + and: 'the error was encountered at the following index in #scenario' + assert exceptionThrown.getDetails().contains("Invalid page index or size") + } } -- cgit 1.2.3-korg