Support pagination in query across all anchors(ep4)

Add pagination query parameters in query across all anchors API
pagination parameters (pageIndex and pageSize) are optional
default is to query all fragments
each pageSize represents number of records(number of anchors)
TotalRecords is returned in response header to find number of pages.

- If pagination option is provided in request then query number of
  anchors equal to pageSize. pageIndex is used for setting offset.

- return number of records(one anchor per record) as per pagesize
  and pageSize

Issue-ID: CPS-1605
Change-ID: I73f97f986a817d423f93a8d922dcd9647b2504bc
Signed-off-by: rajesh.kumar <rk00747546@techmahindra.com>

7 files changed, 159 insertions, 21 deletions

diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java
index f4afe3d73b..19302d67aa 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java
@@ -23,7 +23,8 @@
 
 package org.onap.cps.spi.impl;
 
-import com.google.common.base.Strings;
+import static org.onap.cps.spi.PaginationOption.NO_PAGINATION;
+
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableSet.Builder;
 import io.micrometer.core.annotation.Timed;
@@ -48,6 +49,7 @@ import org.onap.cps.cpspath.parser.CpsPathUtil;
 import org.onap.cps.cpspath.parser.PathParsingException;
 import org.onap.cps.spi.CpsDataPersistenceService;
 import org.onap.cps.spi.FetchDescendantsOption;
+import org.onap.cps.spi.PaginationOption;
 import org.onap.cps.spi.entities.AnchorEntity;
 import org.onap.cps.spi.entities.DataspaceEntity;
 import org.onap.cps.spi.entities.FragmentEntity;
@@ -79,8 +81,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
     private final SessionManager sessionManager;
 
     private static final String REG_EX_FOR_OPTIONAL_LIST_INDEX = "(\\[@.+?])?)";
-    private static final String QUERY_ACROSS_ANCHORS = null;
-    private static final AnchorEntity ALL_ANCHORS = null;
 
     @Override
     public void addChildDataNodes(final String dataspaceName, final String anchorName,
@@ -288,8 +288,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
     public List<DataNode> queryDataNodes(final String dataspaceName, final String anchorName, final String cpsPath,
                                          final FetchDescendantsOption fetchDescendantsOption) {
         final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
-        final AnchorEntity anchorEntity = Strings.isNullOrEmpty(anchorName) ? ALL_ANCHORS
-            : anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName);
+        final AnchorEntity anchorEntity = anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName);
         final CpsPathQuery cpsPathQuery;
         try {
             cpsPathQuery = CpsPathUtil.getCpsPathQuery(cpsPath);
@@ -298,28 +297,60 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
         }
 
         Collection<FragmentEntity> fragmentEntities;
-        if (anchorEntity == ALL_ANCHORS) {
-            fragmentEntities = fragmentRepository.findByDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery);
+        fragmentEntities = fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery);
+        if (cpsPathQuery.hasAncestorAxis()) {
+            final Collection<String> ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery);
+            fragmentEntities = fragmentRepository.findByAnchorAndXpathIn(anchorEntity, ancestorXpaths);
+        }
+        fragmentEntities = fragmentRepository.prefetchDescendantsOfFragmentEntities(fetchDescendantsOption,
+                fragmentEntities);
+        return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities);
+    }
+
+    @Override
+    @Timed(value = "cps.data.persistence.service.datanode.query.anchors",
+            description = "Time taken to query data nodes across all anchors or list of anchors")
+    public List<DataNode> queryDataNodesAcrossAnchors(final String dataspaceName, final String cpsPath,
+                                                      final FetchDescendantsOption fetchDescendantsOption,
+                                                      final PaginationOption paginationOption) {
+        final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
+        final CpsPathQuery cpsPathQuery;
+        try {
+            cpsPathQuery = CpsPathUtil.getCpsPathQuery(cpsPath);
+        } catch (final PathParsingException e) {
+            throw new CpsPathException(e.getMessage());
+        }
+
+        final List<Long> anchorIds;
+        if (paginationOption == NO_PAGINATION) {
+            anchorIds = Collections.EMPTY_LIST;
         } else {
-            fragmentEntities = fragmentRepository.findByAnchorAndCpsPath(anchorEntity, cpsPathQuery);
+            anchorIds = getAnchorIdsForPagination(dataspaceEntity, cpsPathQuery, paginationOption);
+            if (anchorIds.isEmpty()) {
+                return Collections.emptyList();
+            }
         }
+        Collection<FragmentEntity> fragmentEntities =
+            fragmentRepository.findByDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery, anchorIds);
+
         if (cpsPathQuery.hasAncestorAxis()) {
             final Collection<String> ancestorXpaths = processAncestorXpath(fragmentEntities, cpsPathQuery);
-            if (anchorEntity == ALL_ANCHORS) {
+            if (anchorIds.isEmpty()) {
                 fragmentEntities = fragmentRepository.findByDataspaceAndXpathIn(dataspaceEntity, ancestorXpaths);
             } else {
-                fragmentEntities = fragmentRepository.findByAnchorAndXpathIn(anchorEntity, ancestorXpaths);
+                fragmentEntities = fragmentRepository.findByAnchorIdsAndXpathIn(
+                        anchorIds.toArray(new Long[0]), ancestorXpaths.toArray(new String[0]));
             }
+
         }
         fragmentEntities = fragmentRepository.prefetchDescendantsOfFragmentEntities(fetchDescendantsOption,
                 fragmentEntities);
         return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities);
     }
 
-    @Override
-    public List<DataNode> queryDataNodesAcrossAnchors(final String dataspaceName, final String cpsPath,
-                                                      final FetchDescendantsOption fetchDescendantsOption) {
-        return queryDataNodes(dataspaceName, QUERY_ACROSS_ANCHORS, cpsPath, fetchDescendantsOption);
+    private List<Long> getAnchorIdsForPagination(final DataspaceEntity dataspaceEntity, final CpsPathQuery cpsPathQuery,
+                                                 final PaginationOption paginationOption) {
+        return fragmentRepository.findAnchorIdsForPagination(dataspaceEntity, cpsPathQuery, paginationOption);
     }
 
     private List<DataNode> createDataNodesFromFragmentEntities(final FetchDescendantsOption fetchDescendantsOption,
@@ -358,6 +389,19 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService
         sessionManager.lockAnchor(sessionId, dataspaceName, anchorName, timeoutInMilliseconds);
     }
 
+    @Override
+    public Integer countAnchorsForDataspaceAndCpsPath(final String dataspaceName, final String cpsPath) {
+        final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName);
+        final CpsPathQuery cpsPathQuery;
+        try {
+            cpsPathQuery = CpsPathUtil.getCpsPathQuery(cpsPath);
+        } catch (final PathParsingException e) {
+            throw new CpsPathException(e.getMessage());
+        }
+        final List<Long> anchorIdList = getAnchorIdsForPagination(dataspaceEntity, cpsPathQuery, NO_PAGINATION);
+        return anchorIdList.size();
+    }
+
     private static Set<String> processAncestorXpath(final Collection<FragmentEntity> fragmentEntities,
                                                     final CpsPathQuery cpsPathQuery) {
         final Set<String> ancestorXpath = new HashSet<>();
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java
index 1f61ee3c51..c727388b25 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/utils/CpsValidatorImpl.java
@@ -25,6 +25,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.onap.cps.spi.PaginationOption;
 import org.onap.cps.spi.exceptions.DataValidationException;
 import org.onap.cps.spi.utils.CpsValidator;
 import org.springframework.stereotype.Component;
@@ -54,4 +55,16 @@ public class CpsValidatorImpl implements CpsValidator {
             }
         }
     }
+
+    @Override
+    public void validatePaginationOption(final PaginationOption paginationOption) {
+        if (PaginationOption.NO_PAGINATION == paginationOption) {
+            return;
+        }
+
+        if (!paginationOption.isValidPaginationOption()) {
+            throw new DataValidationException("Pagination validation error.",
+                    "Invalid page index or size");
+        }
+    }
 }
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java
index e371035ba5..0c43d62bc7 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentQueryBuilder.java
@@ -21,8 +21,10 @@
 
 package org.onap.cps.spi.repository;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 import java.util.Queue;
 import javax.persistence.EntityManager;
@@ -32,6 +34,7 @@ import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.onap.cps.cpspath.parser.CpsPathPrefixType;
 import org.onap.cps.cpspath.parser.CpsPathQuery;
+import org.onap.cps.spi.PaginationOption;
 import org.onap.cps.spi.entities.AnchorEntity;
 import org.onap.cps.spi.entities.DataspaceEntity;
 import org.onap.cps.spi.entities.FragmentEntity;
@@ -56,7 +59,8 @@ public class FragmentQueryBuilder {
      * @return a executable query object
      */
     public Query getQueryForAnchorAndCpsPath(final AnchorEntity anchorEntity, final CpsPathQuery cpsPathQuery) {
-        return getQueryForDataspaceOrAnchorAndCpsPath(anchorEntity.getDataspace(), anchorEntity, cpsPathQuery);
+        return getQueryForDataspaceOrAnchorAndCpsPath(anchorEntity.getDataspace(),
+                anchorEntity, cpsPathQuery, Collections.EMPTY_LIST);
     }
 
     /**
@@ -67,13 +71,45 @@ public class FragmentQueryBuilder {
      * @return a executable query object
      */
     public Query getQueryForDataspaceAndCpsPath(final DataspaceEntity dataspaceEntity,
-                                                final CpsPathQuery cpsPathQuery) {
-        return getQueryForDataspaceOrAnchorAndCpsPath(dataspaceEntity, ACROSS_ALL_ANCHORS, cpsPathQuery);
+                                                final CpsPathQuery cpsPathQuery,
+                                                final List<Long> anchorIdsForPagination) {
+        return getQueryForDataspaceOrAnchorAndCpsPath(dataspaceEntity, ACROSS_ALL_ANCHORS,
+                cpsPathQuery, anchorIdsForPagination);
+    }
+
+    /**
+     * Get query for dataspace, cps path, page index and page size.
+     * @param dataspaceEntity data space entity
+     * @param cpsPathQuery cps path query
+     * @param paginationOption pagination option
+     * @return query for given dataspace, cps path and pagination parameters
+     */
+    public Query getQueryForAnchorIdsForPagination(final DataspaceEntity dataspaceEntity,
+                                                   final CpsPathQuery cpsPathQuery,
+                                                   final PaginationOption paginationOption) {
+        final StringBuilder sqlStringBuilder = new StringBuilder();
+        final Map<String, Object> queryParameters = new HashMap<>();
+        sqlStringBuilder.append("SELECT distinct(fragment.anchor_id) FROM fragment "
+                + "JOIN anchor ON anchor.id = fragment.anchor_id WHERE dataspace_id = :dataspaceId");
+        queryParameters.put("dataspaceId", dataspaceEntity.getId());
+        addXpathSearch(cpsPathQuery, sqlStringBuilder, queryParameters);
+        addLeafConditions(cpsPathQuery, sqlStringBuilder);
+        addTextFunctionCondition(cpsPathQuery, sqlStringBuilder, queryParameters);
+        addContainsFunctionCondition(cpsPathQuery, sqlStringBuilder, queryParameters);
+        if (PaginationOption.NO_PAGINATION != paginationOption) {
+            sqlStringBuilder.append(" ORDER BY fragment.anchor_id");
+            addPaginationCondition(sqlStringBuilder, queryParameters, paginationOption);
+        }
+
+        final Query query = entityManager.createNativeQuery(sqlStringBuilder.toString());
+        setQueryParameters(query, queryParameters);
+        return query;
     }
 
     private Query getQueryForDataspaceOrAnchorAndCpsPath(final DataspaceEntity dataspaceEntity,
                                                          final AnchorEntity anchorEntity,
-                                                         final CpsPathQuery cpsPathQuery) {
+                                                         final CpsPathQuery cpsPathQuery,
+                                                         final List<Long> anchorIdsForPagination) {
         final StringBuilder sqlStringBuilder = new StringBuilder();
         final Map<String, Object> queryParameters = new HashMap<>();
 
@@ -81,6 +117,10 @@ public class FragmentQueryBuilder {
             sqlStringBuilder.append("SELECT fragment.* FROM fragment JOIN anchor ON anchor.id = fragment.anchor_id"
                 + " WHERE dataspace_id = :dataspaceId");
             queryParameters.put("dataspaceId", dataspaceEntity.getId());
+            if (!anchorIdsForPagination.isEmpty()) {
+                sqlStringBuilder.append(" AND anchor_id IN (:anchorIdsForPagination)");
+                queryParameters.put("anchorIdsForPagination", anchorIdsForPagination);
+            }
         } else {
             sqlStringBuilder.append("SELECT * FROM fragment WHERE anchor_id = :anchorId");
             queryParameters.put("anchorId", anchorEntity.getId());
@@ -107,6 +147,15 @@ public class FragmentQueryBuilder {
         }
     }
 
+    private static void addPaginationCondition(final StringBuilder sqlStringBuilder,
+                                               final Map<String, Object> queryParameters,
+                                               final PaginationOption paginationOption) {
+        final Integer offset = (paginationOption.getPageIndex() - 1) * paginationOption.getPageSize();
+        sqlStringBuilder.append(" LIMIT :pageSize OFFSET :offset");
+        queryParameters.put("pageSize", paginationOption.getPageSize());
+        queryParameters.put("offset", offset);
+    }
+
     private static Integer getTextValueAsInt(final CpsPathQuery cpsPathQuery) {
         try {
             return Integer.parseInt(cpsPathQuery.getTextFunctionConditionValue());
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java
index 303af5bc47..11b2b07733 100755
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java
@@ -68,6 +68,11 @@ public interface FragmentRepository extends JpaRepository<FragmentEntity, Long>,
         return findByDataspaceIdAndXpathIn(dataspaceEntity.getId(), xpaths.toArray(new String[0]));

     }

 

+    @Query(value = "SELECT * FROM fragment WHERE anchor_id IN (:anchorIds)"

+            + " AND xpath = ANY (:xpaths)", nativeQuery = true)

+    List<FragmentEntity> findByAnchorIdsAndXpathIn(@Param("anchorIds") Long[] anchorIds,

+                                                   @Param("xpaths") String[] xpaths);

+

     @Query(value = "SELECT * FROM fragment WHERE anchor_id = :anchorId LIMIT 1", nativeQuery = true)

     Optional<FragmentEntity> findOneByAnchorId(@Param("anchorId") long anchorId);

 

diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java
index de0c060148..9c279618b0 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQuery.java
@@ -23,6 +23,7 @@ package org.onap.cps.spi.repository;
 
 import java.util.List;
 import org.onap.cps.cpspath.parser.CpsPathQuery;
+import org.onap.cps.spi.PaginationOption;
 import org.onap.cps.spi.entities.AnchorEntity;
 import org.onap.cps.spi.entities.DataspaceEntity;
 import org.onap.cps.spi.entities.FragmentEntity;
@@ -30,5 +31,10 @@ import org.onap.cps.spi.entities.FragmentEntity;
 public interface FragmentRepositoryCpsPathQuery {
     List<FragmentEntity> findByAnchorAndCpsPath(AnchorEntity anchorEntity, CpsPathQuery cpsPathQuery);
 
-    List<FragmentEntity> findByDataspaceAndCpsPath(DataspaceEntity dataspaceEntity, CpsPathQuery cpsPathQuery);
+    List<FragmentEntity> findByDataspaceAndCpsPath(DataspaceEntity dataspaceEntity,
+                                                   CpsPathQuery cpsPathQuery, List<Long> anchorIds);
+
+    List<Long> findAnchorIdsForPagination(DataspaceEntity dataspaceEntity, CpsPathQuery cpsPathQuery,
+                                          PaginationOption paginationOption);
+
 }
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java
index 6cc6495b8d..1ba9d1a4c5 100644
--- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java
+++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryCpsPathQueryImpl.java
@@ -29,6 +29,7 @@ import javax.transaction.Transactional;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.onap.cps.cpspath.parser.CpsPathQuery;
+import org.onap.cps.spi.PaginationOption;
 import org.onap.cps.spi.entities.AnchorEntity;
 import org.onap.cps.spi.entities.DataspaceEntity;
 import org.onap.cps.spi.entities.FragmentEntity;
@@ -55,11 +56,21 @@ public class FragmentRepositoryCpsPathQueryImpl implements FragmentRepositoryCps
     @Override
     @Transactional
     public List<FragmentEntity> findByDataspaceAndCpsPath(final DataspaceEntity dataspaceEntity,
-                                                          final CpsPathQuery cpsPathQuery) {
-        final Query query = fragmentQueryBuilder.getQueryForDataspaceAndCpsPath(dataspaceEntity, cpsPathQuery);
+                                                          final CpsPathQuery cpsPathQuery, final List<Long> anchorIds) {
+        final Query query = fragmentQueryBuilder.getQueryForDataspaceAndCpsPath(
+                dataspaceEntity, cpsPathQuery, anchorIds);
         final List<FragmentEntity> fragmentEntities = query.getResultList();
         log.debug("Fetched {} fragment entities by cps path across all anchors.", fragmentEntities.size());
         return fragmentEntities;
     }
 
+    @Override
+    @Transactional
+    public List<Long> findAnchorIdsForPagination(final DataspaceEntity dataspaceEntity, final CpsPathQuery cpsPathQuery,
+                                                 final PaginationOption paginationOption) {
+        final Query query = fragmentQueryBuilder.getQueryForAnchorIdsForPagination(
+                dataspaceEntity, cpsPathQuery, paginationOption);
+        return query.getResultList();
+    }
+
 }
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy
index 345089c931..8d348443c7 100644
--- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy
+++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/utils/CpsValidatorSpec.groovy
@@ -20,6 +20,7 @@
 
 package org.onap.cps.spi.impl.utils
 
+import org.onap.cps.spi.PaginationOption
 import org.onap.cps.spi.exceptions.DataValidationException
 import spock.lang.Specification
 
@@ -64,4 +65,13 @@ class CpsValidatorSpec extends Specification {
         then: 'a data validation exception is thrown'
             thrown(DataValidationException)
     }
+
+    def 'Validate Pagination option with invalid page index and size.'() {
+        when: 'the pagination option is validated using invalid options'
+            objectUnderTest.validatePaginationOption(new PaginationOption(-5, -2))
+        then: 'a data validation exception is thrown'
+            def exceptionThrown = thrown(DataValidationException)
+        and: 'the error was encountered at the following index in #scenario'
+            assert exceptionThrown.getDetails().contains("Invalid page index or size")
+    }
 }