From 9474e8f257f9a03c80c01cbf3729cd128a43847b Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Thu, 8 Jun 2023 14:37:17 +0100 Subject: Lower memory usage in FragmentRepository Avoid using Spring Data "interface projection" in FragmentRepository. The use of FragmentExtract in FragmentRepository is causing an overhead of around 5 kilobytes per fragment, which is leading to abnormally high memory usage when queries return a large number of nodes. For example, around 250MB of additional memory is needlessly used when fetching 50,000 datanodes. - Remove FragmentExtract interface and FragmentEntityArranger class. - Add FragmentPrefetchRepository, using JdbcTemplate and RowMapper to fetch FragmentEntity descendants in a single SQL query. - Many CpsDataService operations have memory reductions: - queryDataNodes - getDataNodesForMultipleXpaths - updateDataNodesAndDescendants - updateNodeLeaves - and any NCMP methods using the above. Issue-ID: CPS-1716 Signed-off-by: danielhanrahan Change-Id: Ic47a2c9eb34150ed76bd5ce452fe1c9aaf9b4c5c --- cps-ri/pom.xml | 2 +- .../cps/spi/entities/FragmentEntityArranger.java | 97 ---------------- .../org/onap/cps/spi/entities/FragmentExtract.java | 34 ------ .../spi/impl/CpsDataPersistenceServiceImpl.java | 54 ++------- .../spi/repository/FragmentPrefetchRepository.java | 31 +++++ .../repository/FragmentPrefetchRepositoryImpl.java | 127 +++++++++++++++++++++ .../cps/spi/repository/FragmentRepository.java | 57 ++------- .../spi/impl/CpsDataPersistenceServiceSpec.groovy | 56 ++++----- 8 files changed, 197 insertions(+), 261 deletions(-) delete mode 100644 cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentEntityArranger.java delete mode 100644 cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentExtract.java create mode 100644 cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepository.java create mode 100644 cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepositoryImpl.java diff --git a/cps-ri/pom.xml b/cps-ri/pom.xml index 66b89de926..504fce560b 100644 --- a/cps-ri/pom.xml +++ b/cps-ri/pom.xml @@ -33,7 +33,7 @@ cps-ri - 0.34 + 0.32 diff --git a/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentEntityArranger.java b/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentEntityArranger.java deleted file mode 100644 index 697eb8de00..0000000000 --- a/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentEntityArranger.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation - * Modifications Copyright (C) 2023 TechMahindra Ltd. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.spi.entities; - -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; - -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class FragmentEntityArranger { - - /** - * Convert a collection of (related) FragmentExtracts into FragmentEntities (trees) with descendants. - * - * @param anchorEntity the anchor(entity) all the fragments belong to - * @param fragmentExtracts FragmentExtracts to convert - * @return a collection of FragmentEntities (trees) with descendants. - */ - public static Collection toFragmentEntityTrees(final AnchorEntity anchorEntity, - final Collection fragmentExtracts) { - final Map fragmentEntityPerId = new HashMap<>(); - if (fragmentExtracts != null) { - for (final FragmentExtract fragmentExtract : fragmentExtracts) { - final FragmentEntity fragmentEntity = toFragmentEntity(anchorEntity, fragmentExtract); - fragmentEntityPerId.put(fragmentEntity.getId(), fragmentEntity); - } - } - return reuniteChildrenWithTheirParents(fragmentEntityPerId); - } - - /** - * Convert a collection of (related) FragmentExtracts into FragmentEntities (trees) with descendants. - * - * @param anchorEntityPerId the anchor(entities) the fragments belong to - * @param fragmentExtracts FragmentExtracts to convert - * @return a collection of FragmentEntities (trees) with descendants. - */ - public static Collection toFragmentEntityTreesAcrossAnchors( - final Map anchorEntityPerId, final Collection fragmentExtracts) { - final Map fragmentEntityPerId = new HashMap<>(); - for (final FragmentExtract fragmentExtract : fragmentExtracts) { - final AnchorEntity anchorEntity = anchorEntityPerId.get(fragmentExtract.getAnchorId()); - final FragmentEntity fragmentEntity = toFragmentEntity(anchorEntity, fragmentExtract); - fragmentEntityPerId.put(fragmentEntity.getId(), fragmentEntity); - } - return reuniteChildrenWithTheirParents(fragmentEntityPerId); - } - - private static FragmentEntity toFragmentEntity(final AnchorEntity anchorEntity, - final FragmentExtract fragmentExtract) { - final FragmentEntity fragmentEntity = new FragmentEntity(); - fragmentEntity.setAnchor(anchorEntity); - fragmentEntity.setId(fragmentExtract.getId()); - fragmentEntity.setXpath(fragmentExtract.getXpath()); - fragmentEntity.setAttributes(fragmentExtract.getAttributes()); - fragmentEntity.setParentId(fragmentExtract.getParentId()); - fragmentEntity.setChildFragments(new HashSet<>()); - return fragmentEntity; - } - - private static Collection reuniteChildrenWithTheirParents( - final Map fragmentEntityPerId) { - final Collection fragmentEntitiesWithoutParentInResultSet = new HashSet<>(); - for (final FragmentEntity fragmentEntity : fragmentEntityPerId.values()) { - final FragmentEntity parentFragmentEntity = fragmentEntityPerId.get(fragmentEntity.getParentId()); - if (parentFragmentEntity == null) { - fragmentEntitiesWithoutParentInResultSet.add(fragmentEntity); - } else { - parentFragmentEntity.getChildFragments().add(fragmentEntity); - } - } - return fragmentEntitiesWithoutParentInResultSet; - } - -} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentExtract.java b/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentExtract.java deleted file mode 100644 index 50be3c7b7a..0000000000 --- a/cps-ri/src/main/java/org/onap/cps/spi/entities/FragmentExtract.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2022-2023 Nordix Foundation. - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -package org.onap.cps.spi.entities; - -public interface FragmentExtract { - - Long getId(); - - Long getAnchorId(); - - String getXpath(); - - Long getParentId(); - - String getAttributes(); -} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java index 02f723029d..e6e250f082 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java @@ -36,7 +36,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -52,8 +51,6 @@ import org.onap.cps.spi.FetchDescendantsOption; import org.onap.cps.spi.entities.AnchorEntity; import org.onap.cps.spi.entities.DataspaceEntity; import org.onap.cps.spi.entities.FragmentEntity; -import org.onap.cps.spi.entities.FragmentEntityArranger; -import org.onap.cps.spi.entities.FragmentExtract; import org.onap.cps.spi.exceptions.AlreadyDefinedException; import org.onap.cps.spi.exceptions.AlreadyDefinedExceptionBatch; import org.onap.cps.spi.exceptions.ConcurrencyException; @@ -248,7 +245,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService final AnchorEntity anchorEntity = getAnchorEntity(dataspaceName, anchorName); final Collection fragmentEntities = getFragmentEntities(anchorEntity, xpaths, fetchDescendantsOption); - return toDataNodes(fragmentEntities, fetchDescendantsOption); + return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); } private Collection getFragmentEntities(final AnchorEntity anchorEntity, @@ -269,19 +266,16 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService normalizedXpaths.addAll(fragmentRepository.findAllXpathByAnchorAndParentIdIsNull(anchorEntity)); } - final List fragmentExtracts = - fragmentRepository.findExtractsWithDescendants(anchorEntity.getId(), normalizedXpaths, - fetchDescendantsOption.getDepth()); + final List fragmentEntities = fragmentRepository.findByAnchorAndXpathIn(anchorEntity, + normalizedXpaths); - return FragmentEntityArranger.toFragmentEntityTrees(anchorEntity, fragmentExtracts); + return fragmentRepository.prefetchDescendantsOfFragmentEntities(fetchDescendantsOption, fragmentEntities); } private FragmentEntity getFragmentEntity(final AnchorEntity anchorEntity, final String xpath) { final FragmentEntity fragmentEntity; if (isRootXpath(xpath)) { - final List fragmentExtracts = fragmentRepository.findAllExtractsByAnchor(anchorEntity); - fragmentEntity = FragmentEntityArranger.toFragmentEntityTrees(anchorEntity, fragmentExtracts) - .stream().findFirst().orElse(null); + fragmentEntity = fragmentRepository.findOneByAnchorId(anchorEntity.getId()).orElse(null); } else { fragmentEntity = fragmentRepository.getByAnchorAndXpath(anchorEntity, getNormalizedXpath(xpath)); } @@ -320,8 +314,8 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService fragmentEntities = fragmentRepository.findByAnchorAndXpathIn(anchorEntity, ancestorXpaths); } } - fragmentEntities = prefetchDescendantsForFragmentEntities(fetchDescendantsOption, anchorEntity, - fragmentEntities); + fragmentEntities = fragmentRepository.prefetchDescendantsOfFragmentEntities(fetchDescendantsOption, + fragmentEntities); return createDataNodesFromFragmentEntities(fetchDescendantsOption, fragmentEntities); } @@ -331,31 +325,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService return queryDataNodes(dataspaceName, QUERY_ACROSS_ANCHORS, cpsPath, fetchDescendantsOption); } - private Collection prefetchDescendantsForFragmentEntities( - final FetchDescendantsOption fetchDescendantsOption, - final AnchorEntity anchorEntity, - final Collection proxiedFragmentEntities) { - if (FetchDescendantsOption.OMIT_DESCENDANTS.equals(fetchDescendantsOption)) { - return proxiedFragmentEntities; - } - - final List fragmentEntityIds = proxiedFragmentEntities.stream() - .map(FragmentEntity::getId).collect(Collectors.toList()); - - final List fragmentExtracts = - fragmentRepository.findExtractsWithDescendantsByIds(fragmentEntityIds, fetchDescendantsOption.getDepth()); - - if (anchorEntity == ALL_ANCHORS) { - final Collection anchorIds = fragmentExtracts.stream() - .map(FragmentExtract::getAnchorId).collect(Collectors.toSet()); - final List anchorEntities = anchorRepository.findAllById(anchorIds); - final Map anchorEntityPerId = anchorEntities.stream() - .collect(Collectors.toMap(AnchorEntity::getId, Function.identity())); - return FragmentEntityArranger.toFragmentEntityTreesAcrossAnchors(anchorEntityPerId, fragmentExtracts); - } - return FragmentEntityArranger.toFragmentEntityTrees(anchorEntity, fragmentExtracts); - } - private List createDataNodesFromFragmentEntities(final FetchDescendantsOption fetchDescendantsOption, final Collection fragmentEntities) { final List dataNodes = new ArrayList<>(fragmentEntities.size()); @@ -422,15 +391,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService .withChildDataNodes(childDataNodes).build(); } - private Collection toDataNodes(final Collection fragmentEntities, - final FetchDescendantsOption fetchDescendantsOption) { - final Collection dataNodes = new ArrayList<>(fragmentEntities.size()); - for (final FragmentEntity fragmentEntity : fragmentEntities) { - dataNodes.add(toDataNode(fragmentEntity, fetchDescendantsOption)); - } - return dataNodes; - } - private List getChildDataNodes(final FragmentEntity fragmentEntity, final FetchDescendantsOption fetchDescendantsOption) { if (fetchDescendantsOption.hasNext()) { diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepository.java new file mode 100644 index 0000000000..2460db869a --- /dev/null +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepository.java @@ -0,0 +1,31 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.spi.repository; + +import java.util.Collection; +import org.onap.cps.spi.FetchDescendantsOption; +import org.onap.cps.spi.entities.FragmentEntity; + +public interface FragmentPrefetchRepository { + Collection prefetchDescendantsOfFragmentEntities( + final FetchDescendantsOption fetchDescendantsOption, + final Collection proxiedFragmentEntities); +} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepositoryImpl.java new file mode 100644 index 0000000000..4f056c8f6e --- /dev/null +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentPrefetchRepositoryImpl.java @@ -0,0 +1,127 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation. + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.spi.repository; + +import java.sql.Connection; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import org.onap.cps.spi.FetchDescendantsOption; +import org.onap.cps.spi.entities.AnchorEntity; +import org.onap.cps.spi.entities.FragmentEntity; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementSetter; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +@Repository +@RequiredArgsConstructor +public class FragmentPrefetchRepositoryImpl implements FragmentPrefetchRepository { + + private final JdbcTemplate jdbcTemplate; + + @Override + public Collection prefetchDescendantsOfFragmentEntities( + final FetchDescendantsOption fetchDescendantsOption, + final Collection proxiedFragmentEntities) { + + if (FetchDescendantsOption.OMIT_DESCENDANTS.equals(fetchDescendantsOption)) { + return proxiedFragmentEntities; + } + + final List fragmentEntityIds = proxiedFragmentEntities.stream() + .map(FragmentEntity::getId).collect(Collectors.toList()); + + final Map anchorEntityPerId = proxiedFragmentEntities.stream() + .map(FragmentEntity::getAnchor) + .collect(Collectors.toMap(AnchorEntity::getId, anchor -> anchor, (anchor1, anchor2) -> anchor1)); + + final int maxDepth = fetchDescendantsOption.equals(FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) + ? Integer.MAX_VALUE + : fetchDescendantsOption.getDepth(); + return findFragmentEntitiesWithDescendantsByIds(fragmentEntityIds, anchorEntityPerId, maxDepth); + } + + private Collection findFragmentEntitiesWithDescendantsByIds( + final Collection fragmentEntityIds, + final Map anchorEntityPerId, + final int maxDepth) { + final String sql + = "WITH RECURSIVE parent_search AS (" + + " SELECT id, 0 AS depth " + + " FROM fragment " + + " WHERE id = ANY (?) " + + " UNION " + + " SELECT child.id, depth + 1 " + + " FROM fragment child INNER JOIN parent_search parent ON child.parent_id = parent.id" + + " WHERE depth < ?" + + ") " + + "SELECT fragment.id, anchor_id AS anchorId, xpath, parent_id AS parentId, " + + " CAST(attributes AS TEXT) AS attributes " + + "FROM fragment INNER JOIN parent_search ON fragment.id = parent_search.id"; + + final PreparedStatementSetter preparedStatementSetter = preparedStatement -> { + final Connection connection = preparedStatement.getConnection(); + final java.sql.Array idArray = connection.createArrayOf("bigint", fragmentEntityIds.toArray()); + preparedStatement.setArray(1, idArray); + preparedStatement.setInt(2, maxDepth); + }; + + final RowMapper fragmentEntityRowMapper = (resultSet, rowNum) -> { + final FragmentEntity fragmentEntity = new FragmentEntity(); + fragmentEntity.setId(resultSet.getLong("id")); + fragmentEntity.setXpath(resultSet.getString("xpath")); + fragmentEntity.setParentId(resultSet.getLong("parentId")); + fragmentEntity.setAttributes(resultSet.getString("attributes")); + fragmentEntity.setAnchor(anchorEntityPerId.get(resultSet.getLong("anchorId"))); + fragmentEntity.setChildFragments(new HashSet<>()); + return fragmentEntity; + }; + + final Map fragmentEntityPerId; + try (final Stream fragmentEntityStream = jdbcTemplate.queryForStream(sql, + preparedStatementSetter, fragmentEntityRowMapper)) { + fragmentEntityPerId = fragmentEntityStream.collect( + Collectors.toMap(FragmentEntity::getId, Function.identity())); + } + return reuniteChildrenWithTheirParents(fragmentEntityPerId); + } + + private static Collection reuniteChildrenWithTheirParents( + final Map fragmentEntityPerId) { + final Collection fragmentEntitiesWithoutParent = new HashSet<>(); + for (final FragmentEntity fragmentEntity : fragmentEntityPerId.values()) { + final FragmentEntity parentFragmentEntity = fragmentEntityPerId.get(fragmentEntity.getParentId()); + if (parentFragmentEntity == null) { + fragmentEntitiesWithoutParent.add(fragmentEntity); + } else { + parentFragmentEntity.getChildFragments().add(fragmentEntity); + } + } + return fragmentEntitiesWithoutParent; + } + +} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java index 82c422f6fd..03de95eb8d 100755 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java @@ -29,7 +29,6 @@ import java.util.Optional; import org.onap.cps.spi.entities.AnchorEntity; import org.onap.cps.spi.entities.DataspaceEntity; import org.onap.cps.spi.entities.FragmentEntity; -import org.onap.cps.spi.entities.FragmentExtract; import org.onap.cps.spi.exceptions.DataNodeNotFoundException; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; @@ -38,7 +37,8 @@ import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @Repository -public interface FragmentRepository extends JpaRepository, FragmentRepositoryCpsPathQuery { +public interface FragmentRepository extends JpaRepository, FragmentRepositoryCpsPathQuery, + FragmentPrefetchRepository { Optional findByAnchorAndXpath(AnchorEntity anchorEntity, String xpath); @@ -47,7 +47,10 @@ public interface FragmentRepository extends JpaRepository, new DataNodeNotFoundException(anchorEntity.getDataspace().getName(), anchorEntity.getName(), xpath)); } - List findByAnchorIdAndXpathIn(long anchorId, String[] xpaths); + @Query(value = "SELECT * FROM fragment WHERE anchor_id = :anchorId AND xpath = ANY (:xpaths)", + nativeQuery = true) + List findByAnchorIdAndXpathIn(@Param("anchorId") long anchorId, + @Param("xpaths") String[] xpaths); default List findByAnchorAndXpathIn(final AnchorEntity anchorEntity, final Collection xpaths) { @@ -66,8 +69,8 @@ public interface FragmentRepository extends JpaRepository, boolean existsByAnchorId(long anchorId); - @Query("SELECT f FROM FragmentEntity f WHERE anchor = :anchor") - List findAllExtractsByAnchor(@Param("anchor") AnchorEntity anchorEntity); + @Query(value = "SELECT * FROM fragment WHERE anchor_id = :anchorId LIMIT 1", nativeQuery = true) + Optional findOneByAnchorId(@Param("anchorId") long anchorId); @Modifying @Query(value = "DELETE FROM fragment WHERE anchor_id = ANY (:anchorIds)", nativeQuery = true) @@ -111,48 +114,4 @@ public interface FragmentRepository extends JpaRepository, @Query("SELECT xpath FROM FragmentEntity WHERE anchor = :anchor AND parentId IS NULL") List findAllXpathByAnchorAndParentIdIsNull(@Param("anchor") AnchorEntity anchorEntity); - @Query(value - = "WITH RECURSIVE parent_search AS (" - + " SELECT id, 0 AS depth " - + " FROM fragment " - + " WHERE anchor_id = :anchorId AND xpath = ANY (:xpaths) " - + " UNION " - + " SELECT c.id, depth + 1 " - + " FROM fragment c INNER JOIN parent_search p ON c.parent_id = p.id" - + " WHERE depth < (SELECT CASE WHEN :maxDepth = -1 THEN " + Integer.MAX_VALUE + " ELSE :maxDepth END) " - + ") " - + "SELECT f.id, anchor_id AS anchorId, xpath, f.parent_id AS parentId, CAST(attributes AS TEXT) AS attributes " - + "FROM fragment f INNER JOIN parent_search p ON f.id = p.id", - nativeQuery = true - ) - List findExtractsWithDescendants(@Param("anchorId") long anchorId, - @Param("xpaths") String[] xpaths, - @Param("maxDepth") int maxDepth); - - default List findExtractsWithDescendants(final long anchorId, final Collection xpaths, - final int maxDepth) { - return findExtractsWithDescendants(anchorId, xpaths.toArray(new String[0]), maxDepth); - } - - @Query(value - = "WITH RECURSIVE parent_search AS (" - + " SELECT id, 0 AS depth " - + " FROM fragment " - + " WHERE id = ANY (:ids) " - + " UNION " - + " SELECT c.id, depth + 1 " - + " FROM fragment c INNER JOIN parent_search p ON c.parent_id = p.id" - + " WHERE depth < (SELECT CASE WHEN :maxDepth = -1 THEN " + Integer.MAX_VALUE + " ELSE :maxDepth END) " - + ") " - + "SELECT f.id, anchor_id AS anchorId, xpath, f.parent_id AS parentId, CAST(attributes AS TEXT) AS attributes " - + "FROM fragment f INNER JOIN parent_search p ON f.id = p.id", - nativeQuery = true - ) - List findExtractsWithDescendantsByIds(@Param("ids") long[] ids, - @Param("maxDepth") int maxDepth); - - default List findExtractsWithDescendantsByIds(final Collection ids, final int maxDepth) { - return findExtractsWithDescendantsByIds(ids.stream().mapToLong(id -> id).toArray(), maxDepth); - } - } diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy index e8921b3ed0..cb554faee8 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy @@ -26,7 +26,7 @@ import org.onap.cps.spi.FetchDescendantsOption import org.onap.cps.spi.entities.AnchorEntity import org.onap.cps.spi.entities.DataspaceEntity import org.onap.cps.spi.entities.FragmentEntity -import org.onap.cps.spi.entities.FragmentExtract + import org.onap.cps.spi.exceptions.ConcurrencyException import org.onap.cps.spi.exceptions.DataValidationException import org.onap.cps.spi.model.DataNode @@ -55,6 +55,7 @@ class CpsDataPersistenceServiceSpec extends Specification { def setup() { mockAnchorRepository.getByDataspaceAndName(_, _) >> anchorEntity + mockFragmentRepository.prefetchDescendantsOfFragmentEntities(_, _) >> { fetchDescendantsOption, fragmentEntities -> fragmentEntities } } def 'Storing data nodes individually when batch operation fails'(){ @@ -93,20 +94,20 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Batch update data node leaves and descendants: #scenario'(){ given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findExtractsWithDescendants(_, [] as Set, _) >> [] - mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath', "{\"id\":\"testId1\"}") + mockFragmentRepository.findByAnchorAndXpathIn(_, [] as Set) >> [] + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath'] as Set) >> [ + new FragmentEntity(1, '/test/xpath', null, "{\"id\":\"testId\"}", anchorEntity, [] as Set) ] - mockFragmentRepository.findExtractsWithDescendants(123, ['/test/xpath1', '/test/xpath2'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath1', "{\"id\":\"testId1\"}"), - mockFragmentExtract(2, null, 123, '/test/xpath2', "{\"id\":\"testId1\"}") + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath1', '/test/xpath2'] as Set) >> [ + new FragmentEntity(1, '/test/xpath1', null, "{\"id\":\"testId1\"}", anchorEntity, [] as Set), + new FragmentEntity(2, '/test/xpath2', null, "{\"id\":\"testId2\"}", anchorEntity, [] as Set) ] when: 'replace data node tree' objectUnderTest.batchUpdateDataLeaves('dataspaceName', 'anchorName', dataNodes.stream().collect(Collectors.toMap(DataNode::getXpath, DataNode::getLeaves))) then: 'call fragment repository save all method' 1 * mockFragmentRepository.saveAll({fragmentEntities -> - assert fragmentEntities as List == expectedFragmentEntities + assert fragmentEntities.sort() == expectedFragmentEntities.sort() assert fragmentEntities.size() == expectedSize }) where: 'the following Data Type is passed' @@ -172,9 +173,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Retrieving multiple data nodes.'() { given: 'fragment repository returns a collection of fragments' - mockFragmentRepository.findExtractsWithDescendants(123, ['/xpath1', '/xpath2'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/xpath1', null), - mockFragmentExtract(2, null, 123, '/xpath2', null) + mockFragmentRepository.findByAnchorAndXpathIn(anchorEntity, ['/xpath1', '/xpath2'] as Set) >> [ + new FragmentEntity(1, '/xpath1', null, null, anchorEntity, [] as Set), + new FragmentEntity(2, '/xpath2', null, null, anchorEntity, [] as Set) ] when: 'getting data nodes for 2 xpaths' def result = objectUnderTest.getDataNodesForMultipleXpaths('some-dataspace', 'some-anchor', ['/xpath1', '/xpath2'], FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) @@ -207,9 +208,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Replace data node and descendants: #scenario'(){ given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findExtractsWithDescendants(_, [] as Set, _) >> [] - mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath', null) + mockFragmentRepository.findByAnchorAndXpathIn(_, [] as Set) >> [] + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath'] as Set) >> [ + new FragmentEntity(1, '/test/xpath', null, '{"id":"testId"}', anchorEntity, [] as Set) ] when: 'replace data node tree' objectUnderTest.updateDataNodesAndDescendants('dataspaceName', 'anchorName', dataNodes) @@ -223,9 +224,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Replace data nodes and descendants'() { given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath1', '/test/xpath2'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath1', null), - mockFragmentExtract(2, null, 123, '/test/xpath2', null) + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath1', '/test/xpath2'] as Set) >> [ + new FragmentEntity(1, '/test/xpath1', null, null, anchorEntity, [] as Set), + new FragmentEntity(2, '/test/xpath2', null, null, anchorEntity, [] as Set) ] and: 'some data nodes with descendants' def dataNode1 = new DataNode(xpath: '/test/xpath1', leaves: ['id': 'testId1'], childDataNodes: [new DataNode(xpath: '/test/xpath1/child', leaves: ['id': 'childTestId1'])]) @@ -253,38 +254,27 @@ class CpsDataPersistenceServiceSpec extends Specification { def createDataNodesAndMockRepositoryMethodSupportingThem(Map xpathToScenarioMap) { def dataNodes = [] - def fragmentExtracts = [] + def fragmentEntities = [] def fragmentId = 1 xpathToScenarioMap.each { def xpath = it.key def scenario = it.value def dataNode = new DataNodeBuilder().withXpath(xpath).build() dataNodes.add(dataNode) - def fragmentExtract = mockFragmentExtract(fragmentId, null, 123, xpath, null) - fragmentExtracts.add(fragmentExtract) def fragmentEntity = new FragmentEntity(id: fragmentId, anchor: anchorEntity, xpath: xpath, childFragments: []) + fragmentEntities.add(fragmentEntity) if ('EXCEPTION' == scenario) { mockFragmentRepository.save(fragmentEntity) >> { throw new StaleStateException("concurrent updates") } } fragmentId++ } - mockFragmentRepository.findExtractsWithDescendants(_, xpathToScenarioMap.keySet(), _) >> fragmentExtracts + mockFragmentRepository.findByAnchorAndXpathIn(_, xpathToScenarioMap.keySet()) >> fragmentEntities return dataNodes } def mockFragmentWithJson(json) { - def fragmentExtract = mockFragmentExtract(456, null, 123, '/parent-01', json) - mockFragmentRepository.findExtractsWithDescendants(123, ['/parent-01'] as Set, _) >> [fragmentExtract] - } - - def mockFragmentExtract(id, parentId, anchorId, xpath, attributes) { - def fragmentExtract = Mock(FragmentExtract) - fragmentExtract.getId() >> id - fragmentExtract.getParentId() >> parentId - fragmentExtract.getAnchorId() >> anchorId - fragmentExtract.getXpath() >> xpath - fragmentExtract.getAttributes() >> attributes - return fragmentExtract + def fragmentEntity = new FragmentEntity(456, '/parent-01', null, json, anchorEntity, [] as Set) + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/parent-01'] as Set) >> [fragmentEntity] } } -- cgit 1.2.3-korg