diff options
author | 2023-01-24 11:23:02 +0000 | |
---|---|---|
committer | 2023-02-01 13:56:18 +0000 | |
commit | 230b1119dec71e301ba462246c3fc53d0fc0281a (patch) | |
tree | 8f889414b78aa4c750fd9084f57ab13f0a5a156b /cps-ri/src/main | |
parent | 447c872eb3c9bd57631127651bc9744c5c1a8643 (diff) |
Create plural version of deleteDataNode
- Add method to CpsDataService to batch delete data nodes and lists
- Use native queries to batch delete fragment entities by xpaths,
for data nodes and lists
- Add performance tests for batch delete
- Refactor FragmentNativeRepository
- Add single-column version of createTemporaryTable
- Renamed metric cps.data.service.datanode.batch.delete
to cps.data.service.datanode.all.delete
Issue-ID: CPS-1438
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: I1851f9c7ef0b1be4bd421b3352d9697a2dd23f79
Diffstat (limited to 'cps-ri/src/main')
6 files changed, 123 insertions, 45 deletions
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java index d2b7273fe..5b310efd5 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java @@ -61,7 +61,6 @@ import org.onap.cps.spi.model.DataNode; import org.onap.cps.spi.model.DataNodeBuilder; import org.onap.cps.spi.repository.AnchorRepository; import org.onap.cps.spi.repository.DataspaceRepository; -import org.onap.cps.spi.repository.FragmentNativeRepository; import org.onap.cps.spi.repository.FragmentQueryBuilder; import org.onap.cps.spi.repository.FragmentRepository; import org.onap.cps.spi.utils.SessionManager; @@ -79,7 +78,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService private final FragmentRepository fragmentRepository; private final JsonObjectMapper jsonObjectMapper; private final SessionManager sessionManager; - private final FragmentNativeRepository fragmentNativeRepositoryImpl; private static final String REG_EX_FOR_OPTIONAL_LIST_INDEX = "(\\[@[\\s\\S]+?]){0,1})"; @@ -609,6 +607,26 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService @Override @Transactional + public void deleteDataNodes(final String dataspaceName, final String anchorName, + final Collection<String> xpathsToDelete) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final AnchorEntity anchorEntity = anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName); + + final Collection<String> normalizedXpaths = new ArrayList<>(xpathsToDelete.size()); + for (final String xpath : xpathsToDelete) { + try { + normalizedXpaths.add(CpsPathUtil.getNormalizedXpath(xpath)); + } catch (final PathParsingException e) { + log.debug("Error parsing xpath \"{}\" in deleteDataNodes: {}", xpath, e.getMessage()); + } + } + + fragmentRepository.deleteByAnchorIdAndXpaths(anchorEntity.getId(), normalizedXpaths); + fragmentRepository.deleteListsByAnchorIdAndXpaths(anchorEntity.getId(), normalizedXpaths); + } + + @Override + @Transactional public void deleteListDataNode(final String dataspaceName, final String anchorName, final String targetXpath) { deleteDataNode(dataspaceName, anchorName, targetXpath, true); @@ -656,7 +674,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService private boolean deleteDataNode(final FragmentEntity parentFragmentEntity, final String targetXpath) { final String normalizedTargetXpath = CpsPathUtil.getNormalizedXpath(targetXpath); if (parentFragmentEntity.getXpath().equals(normalizedTargetXpath)) { - fragmentNativeRepositoryImpl.deleteFragmentEntity(parentFragmentEntity.getId()); + fragmentRepository.deleteFragmentEntity(parentFragmentEntity.getId()); return true; } if (parentFragmentEntity.getChildFragments() diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java index 4cfd79dee..13320bf76 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java @@ -20,9 +20,31 @@ package org.onap.cps.spi.repository; +import java.util.Collection; + /** * This interface is used in delete fragment entity by id with child using native sql queries. */ public interface FragmentNativeRepository { void deleteFragmentEntity(long fragmentEntityId); + + /** + * Delete fragment entities for each supplied xpath. + * This method will delete list elements or other data nodes, but not whole lists. + * Non-existing xpaths will not result in an exception. + * @param anchorId the id of the anchor + * @param xpaths xpaths of data nodes to remove + */ + void deleteByAnchorIdAndXpaths(int anchorId, Collection<String> xpaths); + + /** + * Delete fragment entities that are list elements of each supplied list xpath. + * For example, if xpath '/parent/list' is provided, then list all elements in '/parent/list' will be deleted, + * e.g. /parent/list[@key='A'], /parent/list[@key='B']. + * This method will only delete whole lists by xpath; xpaths to list elements or other data nodes will be ignored. + * Non-existing xpaths will not result in an exception. + * @param anchorId the id of the anchor + * @param listXpaths xpaths of whole lists to remove + */ + void deleteListsByAnchorIdAndXpaths(int anchorId, Collection<String> listXpaths); } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java index 57dca568f..0e4d359da 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java @@ -20,13 +20,12 @@ package org.onap.cps.spi.repository; -import java.sql.PreparedStatement; +import java.util.Collection; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.hibernate.Session; -import org.springframework.stereotype.Repository; +import lombok.RequiredArgsConstructor; -@Repository +@RequiredArgsConstructor public class FragmentNativeRepositoryImpl implements FragmentNativeRepository { private static final String DROP_FRAGMENT_CONSTRAINT @@ -34,28 +33,62 @@ public class FragmentNativeRepositoryImpl implements FragmentNativeRepository { private static final String ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE = "ALTER TABLE fragment ADD CONSTRAINT fragment_parent_id_fkey FOREIGN KEY (parent_id) " + "REFERENCES fragment (id) ON DELETE CASCADE;"; - private static final String DELETE_FRAGMENT = "DELETE FROM fragment WHERE id =?;"; private static final String ADD_ORIGINAL_FRAGMENT_CONSTRAINT = "ALTER TABLE fragment ADD CONSTRAINT fragment_parent_id_fkey FOREIGN KEY (parent_id) " + "REFERENCES fragment (id) ON DELETE NO ACTION;"; @PersistenceContext - private EntityManager entityManager; + private final EntityManager entityManager; + + private final TempTableCreator tempTableCreator; @Override public void deleteFragmentEntity(final long fragmentEntityId) { - final Session session = entityManager.unwrap(Session.class); - session.doWork(connection -> { - try (PreparedStatement preparedStatement = connection.prepareStatement( + entityManager.createNativeQuery( + DROP_FRAGMENT_CONSTRAINT + + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE + + "DELETE FROM fragment WHERE id = ?;" + + DROP_FRAGMENT_CONSTRAINT + + ADD_ORIGINAL_FRAGMENT_CONSTRAINT) + .setParameter(1, fragmentEntityId) + .executeUpdate(); + } + + @Override + // Accept security hotspot as temporary table name in SQL query is created internally, not from user input. + @SuppressWarnings("squid:S2077") + public void deleteByAnchorIdAndXpaths(final int anchorId, final Collection<String> xpaths) { + if (!xpaths.isEmpty()) { + final String tempTableName = tempTableCreator.createTemporaryTable("xpathsToDelete", xpaths, "xpath"); + entityManager.createNativeQuery( DROP_FRAGMENT_CONSTRAINT - + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE - + DELETE_FRAGMENT - + DROP_FRAGMENT_CONSTRAINT - + ADD_ORIGINAL_FRAGMENT_CONSTRAINT)) { - preparedStatement.setLong(1, fragmentEntityId); - preparedStatement.executeUpdate(); - } - }); + + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE + + "DELETE FROM fragment f USING " + tempTableName + " t" + + " WHERE f.anchor_id = :anchorId AND f.xpath = t.xpath;" + + DROP_FRAGMENT_CONSTRAINT + + ADD_ORIGINAL_FRAGMENT_CONSTRAINT) + .setParameter("anchorId", anchorId) + .executeUpdate(); + } } -} + @Override + // Accept security hotspot as temporary table name in SQL query is created internally, not from user input. + @SuppressWarnings("squid:S2077") + public void deleteListsByAnchorIdAndXpaths(final int anchorId, final Collection<String> xpaths) { + if (!xpaths.isEmpty()) { + final String tempTableName = tempTableCreator.createTemporaryTable("xpathsToDelete", xpaths, "xpath"); + entityManager.createNativeQuery( + DROP_FRAGMENT_CONSTRAINT + + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE + + "DELETE FROM fragment f USING " + tempTableName + " t" + + " WHERE f.anchor_id = :anchorId AND f.xpath LIKE CONCAT(t.xpath, :xpathListPattern);" + + DROP_FRAGMENT_CONSTRAINT + + ADD_ORIGINAL_FRAGMENT_CONSTRAINT) + .setParameter("anchorId", anchorId) + .setParameter("xpathListPattern", "[%%") + .executeUpdate(); + } + } + +} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java index 4b42b2da8..8bdb7d985 100755 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java @@ -1,6 +1,6 @@ /*
* ============LICENSE_START=======================================================
- * Copyright (C) 2021-2022 Nordix Foundation.
+ * Copyright (C) 2021-2023 Nordix Foundation.
* Modifications Copyright (C) 2020-2021 Bell Canada.
* Modifications Copyright (C) 2020-2021 Pantheon.tech.
* ================================================================================
@@ -40,7 +40,7 @@ import org.springframework.stereotype.Repository; @Repository
public interface FragmentRepository extends JpaRepository<FragmentEntity, Long>, FragmentRepositoryCpsPathQuery,
- FragmentRepositoryMultiPathQuery {
+ FragmentRepositoryMultiPathQuery, FragmentNativeRepository {
Optional<FragmentEntity> findByDataspaceAndAnchorAndXpath(@NonNull DataspaceEntity dataspaceEntity,
@NonNull AnchorEntity anchorEntity,
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java index 8c357bbb3..151fe97b3 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java @@ -20,28 +20,24 @@ package org.onap.cps.spi.repository; - -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.transaction.Transactional; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.spi.entities.FragmentEntity; - @Slf4j -@AllArgsConstructor +@RequiredArgsConstructor public class FragmentRepositoryMultiPathQueryImpl implements FragmentRepositoryMultiPathQuery { @PersistenceContext - private EntityManager entityManager; + private final EntityManager entityManager; - private TempTableCreator tempTableCreator; + private final TempTableCreator tempTableCreator; @Override @Transactional @@ -50,24 +46,13 @@ public class FragmentRepositoryMultiPathQueryImpl implements FragmentRepositoryM if (cpsPathQueryList.isEmpty()) { return Collections.emptyList(); } - final Collection<List<String>> sqlData = new HashSet<>(cpsPathQueryList.size()); - for (final String query : cpsPathQueryList) { - final List<String> row = new ArrayList<>(1); - row.add(query); - sqlData.add(row); - } - final String tempTableName = tempTableCreator.createTemporaryTable( - "xpathTemporaryTable", sqlData, "xpath"); - return selectMatchingFragments(anchorId, tempTableName); - } - - private List<FragmentEntity> selectMatchingFragments(final Integer anchorId, final String tempTableName) { + "xpathTemporaryTable", cpsPathQueryList, "xpath"); final String sql = String.format( "SELECT * FROM FRAGMENT WHERE anchor_id = %d AND xpath IN (select xpath FROM %s);", anchorId, tempTableName); final List<FragmentEntity> fragmentEntities = entityManager.createNativeQuery(sql, FragmentEntity.class) - .getResultList(); + .getResultList(); log.debug("Fetched {} fragment entities by anchor and cps path.", fragmentEntities.size()); return fragmentEntities; } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java index d713746e4..338b0b1c6 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation. + * Copyright (C) 2022-2023 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,10 @@ package org.onap.cps.spi.repository; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -63,6 +65,24 @@ public class TempTableCreator { return tempTableName; } + /** + * Create a uniquely named temporary table with a single column. + * + * @param prefix prefix for the table name (so you can recognize it) + * @param sqlData data to insert (strings only); each entry is a single row of data + * @param columnName column name + * @return a unique temporary table name with given prefix + */ + public String createTemporaryTable(final String prefix, + final Collection<String> sqlData, + final String columnName) { + final Collection<List<String>> tableData = new ArrayList<>(sqlData.size()); + for (final String entry : sqlData) { + tableData.add(Collections.singletonList(entry)); + } + return createTemporaryTable(prefix, tableData, columnName); + } + private static void defineColumns(final StringBuilder sqlStringBuilder, final String[] columnNames) { sqlStringBuilder.append('('); final Iterator<String> it = Arrays.stream(columnNames).iterator(); |