diff options
23 files changed, 400 insertions, 139 deletions
diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java index 8b80a0341a..a5bd60605a 100755 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImpl.java @@ -1,7 +1,7 @@ /* * ============LICENSE_START======================================================= * Copyright (C) 2021 highstreet technologies GmbH - * Modifications Copyright (C) 2021-2022 Nordix Foundation + * Modifications Copyright (C) 2021-2023 Nordix Foundation * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2021-2022 Bell Canada * ================================================================================ @@ -27,6 +27,7 @@ import static org.onap.cps.ncmp.api.impl.constants.DmiRegistryConstants.NFP_OPER import static org.onap.cps.ncmp.api.impl.operations.DmiRequestBody.OperationEnum; import static org.onap.cps.ncmp.api.impl.utils.RestQueryParametersValidator.validateCmHandleQueryParameters; +import com.google.common.collect.Lists; import com.hazelcast.map.IMap; import java.time.OffsetDateTime; import java.util.ArrayList; @@ -78,6 +79,7 @@ import org.springframework.stereotype.Service; @RequiredArgsConstructor public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService { + private static final int DELETE_BATCH_SIZE = 100; private final JsonObjectMapper jsonObjectMapper; private final DmiDataOperations dmiDataOperations; private final NetworkCmProxyDataServicePropertyHandler networkCmProxyDataServicePropertyHandler; @@ -330,25 +332,19 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService .collect(Collectors.toList()); updateCmHandleStateBatch(yangModelCmHandles, CmHandleState.DELETING); - for (final String cmHandleId : tobeRemovedCmHandles) { + for (final List<String> tobeRemovedCmHandleBatch : Lists.partition(tobeRemovedCmHandles, DELETE_BATCH_SIZE)) { try { - deleteCmHandleFromDbAndModuleSyncMap(cmHandleId); - cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId)); - } catch (final DataNodeNotFoundException dataNodeNotFoundException) { - log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}", - cmHandleId, dataNodeNotFoundException.getMessage()); - cmHandleRegistrationResponses.add(CmHandleRegistrationResponse - .createFailureResponse(cmHandleId, RegistrationError.CM_HANDLE_DOES_NOT_EXIST)); - } catch (final DataValidationException dataValidationException) { - log.error("Unable to de-register cm-handle id: {}, caused by: {}", - cmHandleId, dataValidationException.getMessage()); - cmHandleRegistrationResponses.add(CmHandleRegistrationResponse - .createFailureResponse(cmHandleId, RegistrationError.CM_HANDLE_INVALID_ID)); - } catch (final Exception exception) { - log.error("Unable to de-register cm-handle id : {} , caused by : {}", - cmHandleId, exception.getMessage()); - cmHandleRegistrationResponses.add( - CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception)); + batchDeleteCmHandlesFromDbAndModuleSyncMap(tobeRemovedCmHandleBatch); + tobeRemovedCmHandleBatch.forEach(cmHandleId -> + cmHandleRegistrationResponses.add(CmHandleRegistrationResponse.createSuccessResponse(cmHandleId))); + + } catch (final Exception batchException) { + log.error("Unable to de-register cm-handle batch, retrying on each cm handle"); + for (final String cmHandleId : tobeRemovedCmHandleBatch) { + final CmHandleRegistrationResponse cmHandleRegistrationResponse = + deleteCmHandleAndGetCmHandleRegistrationResponse(cmHandleId); + cmHandleRegistrationResponses.add(cmHandleRegistrationResponse); + } } } @@ -356,6 +352,26 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService return cmHandleRegistrationResponses; } + private CmHandleRegistrationResponse deleteCmHandleAndGetCmHandleRegistrationResponse(final String cmHandleId) { + try { + deleteCmHandleFromDbAndModuleSyncMap(cmHandleId); + return CmHandleRegistrationResponse.createSuccessResponse(cmHandleId); + } catch (final DataNodeNotFoundException dataNodeNotFoundException) { + log.error("Unable to find dataNode for cmHandleId : {} , caused by : {}", + cmHandleId, dataNodeNotFoundException.getMessage()); + return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, + RegistrationError.CM_HANDLE_DOES_NOT_EXIST); + } catch (final DataValidationException dataValidationException) { + log.error("Unable to de-register cm-handle id: {}, caused by: {}", + cmHandleId, dataValidationException.getMessage()); + return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, + RegistrationError.CM_HANDLE_INVALID_ID); + } catch (final Exception exception) { + log.error("Unable to de-register cm-handle id : {} , caused by : {}", cmHandleId, exception.getMessage()); + return CmHandleRegistrationResponse.createFailureResponse(cmHandleId, exception); + } + } + private void updateCmHandleStateBatch(final List<YangModelCmHandle> yangModelCmHandles, final CmHandleState cmHandleState) { final Map<YangModelCmHandle, CmHandleState> cmHandleIdsToBeRemoved = new HashMap<>(); @@ -366,10 +382,22 @@ public class NetworkCmProxyDataServiceImpl implements NetworkCmProxyDataService private void deleteCmHandleFromDbAndModuleSyncMap(final String cmHandleId) { inventoryPersistence.deleteSchemaSetWithCascade(cmHandleId); - inventoryPersistence.deleteListOrListElement("/dmi-registry/cm-handles[@id='" + cmHandleId + "']"); + inventoryPersistence.deleteDataNode("/dmi-registry/cm-handles[@id='" + cmHandleId + "']"); removeDeletedCmHandleFromModuleSyncMap(cmHandleId); } + private void batchDeleteCmHandlesFromDbAndModuleSyncMap(final Collection<String> tobeRemovedCmHandles) { + tobeRemovedCmHandles.forEach(inventoryPersistence::deleteSchemaSetWithCascade); + inventoryPersistence.deleteDataNodes(mapCmHandleIdsToXpaths(tobeRemovedCmHandles)); + tobeRemovedCmHandles.forEach(this::removeDeletedCmHandleFromModuleSyncMap); + } + + private Collection<String> mapCmHandleIdsToXpaths(final Collection<String> cmHandles) { + return cmHandles.stream() + .map(cmHandleId -> "/dmi-registry/cm-handles[@id='" + cmHandleId + "']") + .collect(Collectors.toSet()); + } + // CPS-1239 Robustness cleaning of in progress cache private void removeDeletedCmHandleFromModuleSyncMap(final String deletedCmHandleId) { if (moduleSyncStartedOnCmHandles.remove(deletedCmHandleId) != null) { diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java index 6d006d9e2a..55442ecffb 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistence.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation + * Copyright (C) 2022-2023 Nordix Foundation * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -147,9 +147,16 @@ public interface InventoryPersistence { void replaceListContent(String parentNodeXpath, Collection<DataNode> dataNodes); /** - * Deletes data node for given anchor and dataspace. + * Deletes data node. * * @param dataNodeXpath data node xpath */ void deleteDataNode(String dataNodeXpath); + + /** + * Deletes multiple data nodes. + * + * @param dataNodeXpaths data node xpaths + */ + void deleteDataNodes(Collection<String> dataNodeXpaths); } diff --git a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java index dd165853d1..29712f4d0b 100644 --- a/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java +++ b/cps-ncmp-service/src/main/java/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImpl.java @@ -184,7 +184,11 @@ public class InventoryPersistenceImpl implements InventoryPersistence { @Override public void deleteDataNode(final String dataNodeXpath) { - cpsDataService.deleteDataNode(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpath, - NO_TIMESTAMP); + cpsDataService.deleteDataNode(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpath, NO_TIMESTAMP); + } + + @Override + public void deleteDataNodes(final Collection<String> dataNodeXpaths) { + cpsDataService.deleteDataNodes(NCMP_DATASPACE_NAME, NCMP_DMI_REGISTRY_ANCHOR, dataNodeXpaths, NO_TIMESTAMP); } } diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy index 1ebd69eb60..5824a47292 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/impl/NetworkCmProxyDataServiceImplRegistrationSpec.groovy @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2021-2022 Nordix Foundation + * Copyright (C) 2021-2023 Nordix Foundation * Modifications Copyright (C) 2022 Bell Canada * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); @@ -256,7 +256,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { and: 'method to delete relevant schema set is called once' 1 * mockInventoryPersistence.deleteSchemaSetWithCascade(_) and: 'method to delete relevant list/list element is called once' - 1 * mockInventoryPersistence.deleteListOrListElement(_) + 1 * mockInventoryPersistence.deleteDataNodes(_) and: 'successful response is received' assert response.getRemovedCmHandles().size() == 1 with(response.getRemovedCmHandles().get(0)) { @@ -275,8 +275,10 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { given: 'a registration with three cm-handles to be deleted' def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle1', 'cmhandle2', 'cmhandle3']) + and: 'cm-handle deletion fails on batch' + mockInventoryPersistence.deleteDataNodes(_) >> { throw new RuntimeException("Failed") } and: 'cm-handle deletion is successful for 1st and 3rd; failed for 2nd' - mockInventoryPersistence.deleteListOrListElement(_) >> {} >> { throw new RuntimeException("Failed") } >> {} + mockInventoryPersistence.deleteDataNode(_) >> {} >> { throw new RuntimeException("Failed") } >> {} when: 'registration is updated to delete cmhandles' def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration) then: 'a response is received for all cm-handles' @@ -315,7 +317,7 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { then: 'no exception is thrown' noExceptionThrown() and: 'cm-handle is not deleted' - 0 * mockInventoryPersistence.deleteListOrListElement(_) + 0 * mockInventoryPersistence.deleteDataNodes(_) and: 'the cmHandle state is not updated to "DELETED"' 0 * mockLcmEventsCmHandleStateHandler.updateCmHandleState(_, CmHandleState.DELETED) and: 'a failure response is received' @@ -333,7 +335,8 @@ class NetworkCmProxyDataServiceImplRegistrationSpec extends Specification { def dmiPluginRegistration = new DmiPluginRegistration(dmiPlugin: 'my-server', removedCmHandles: ['cmhandle']) and: 'cm-handle deletion throws exception' - mockInventoryPersistence.deleteListOrListElement(_) >> { throw deleteListElementException } + mockInventoryPersistence.deleteDataNodes(_) >> { throw deleteListElementException } + mockInventoryPersistence.deleteDataNode(_) >> { throw deleteListElementException } when: 'registration is updated to delete cmhandle' def response = objectUnderTest.updateDmiRegistrationAndSyncModule(dmiPluginRegistration) then: 'a failure response is received' diff --git a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImplSpec.groovy b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImplSpec.groovy index 355487f64a..2ca0e9964a 100644 --- a/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImplSpec.groovy +++ b/cps-ncmp-service/src/test/groovy/org/onap/cps/ncmp/api/inventory/InventoryPersistenceImplSpec.groovy @@ -270,4 +270,12 @@ class InventoryPersistenceImplSpec extends Specification { 'sample dataNode xpath', NO_TIMESTAMP); } + def 'Delete multiple data nodes via xPath'() { + when: 'Delete data nodes method is called with multiple xpaths as parameters' + objectUnderTest.deleteDataNodes(['xpath1', 'xpath2']) + then: 'the cps data service method to delete data nodes is invoked once with the same xPaths' + 1 * mockCpsDataService.deleteDataNodes('NCMP-Admin', 'ncmp-dmi-registry', + ['xpath1', 'xpath2'], NO_TIMESTAMP); + } + } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java index d2b7273fe1..5b310efd5d 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/impl/CpsDataPersistenceServiceImpl.java @@ -61,7 +61,6 @@ import org.onap.cps.spi.model.DataNode; import org.onap.cps.spi.model.DataNodeBuilder; import org.onap.cps.spi.repository.AnchorRepository; import org.onap.cps.spi.repository.DataspaceRepository; -import org.onap.cps.spi.repository.FragmentNativeRepository; import org.onap.cps.spi.repository.FragmentQueryBuilder; import org.onap.cps.spi.repository.FragmentRepository; import org.onap.cps.spi.utils.SessionManager; @@ -79,7 +78,6 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService private final FragmentRepository fragmentRepository; private final JsonObjectMapper jsonObjectMapper; private final SessionManager sessionManager; - private final FragmentNativeRepository fragmentNativeRepositoryImpl; private static final String REG_EX_FOR_OPTIONAL_LIST_INDEX = "(\\[@[\\s\\S]+?]){0,1})"; @@ -609,6 +607,26 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService @Override @Transactional + public void deleteDataNodes(final String dataspaceName, final String anchorName, + final Collection<String> xpathsToDelete) { + final DataspaceEntity dataspaceEntity = dataspaceRepository.getByName(dataspaceName); + final AnchorEntity anchorEntity = anchorRepository.getByDataspaceAndName(dataspaceEntity, anchorName); + + final Collection<String> normalizedXpaths = new ArrayList<>(xpathsToDelete.size()); + for (final String xpath : xpathsToDelete) { + try { + normalizedXpaths.add(CpsPathUtil.getNormalizedXpath(xpath)); + } catch (final PathParsingException e) { + log.debug("Error parsing xpath \"{}\" in deleteDataNodes: {}", xpath, e.getMessage()); + } + } + + fragmentRepository.deleteByAnchorIdAndXpaths(anchorEntity.getId(), normalizedXpaths); + fragmentRepository.deleteListsByAnchorIdAndXpaths(anchorEntity.getId(), normalizedXpaths); + } + + @Override + @Transactional public void deleteListDataNode(final String dataspaceName, final String anchorName, final String targetXpath) { deleteDataNode(dataspaceName, anchorName, targetXpath, true); @@ -656,7 +674,7 @@ public class CpsDataPersistenceServiceImpl implements CpsDataPersistenceService private boolean deleteDataNode(final FragmentEntity parentFragmentEntity, final String targetXpath) { final String normalizedTargetXpath = CpsPathUtil.getNormalizedXpath(targetXpath); if (parentFragmentEntity.getXpath().equals(normalizedTargetXpath)) { - fragmentNativeRepositoryImpl.deleteFragmentEntity(parentFragmentEntity.getId()); + fragmentRepository.deleteFragmentEntity(parentFragmentEntity.getId()); return true; } if (parentFragmentEntity.getChildFragments() diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java index 4cfd79dee3..13320bf763 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepository.java @@ -20,9 +20,31 @@ package org.onap.cps.spi.repository; +import java.util.Collection; + /** * This interface is used in delete fragment entity by id with child using native sql queries. */ public interface FragmentNativeRepository { void deleteFragmentEntity(long fragmentEntityId); + + /** + * Delete fragment entities for each supplied xpath. + * This method will delete list elements or other data nodes, but not whole lists. + * Non-existing xpaths will not result in an exception. + * @param anchorId the id of the anchor + * @param xpaths xpaths of data nodes to remove + */ + void deleteByAnchorIdAndXpaths(int anchorId, Collection<String> xpaths); + + /** + * Delete fragment entities that are list elements of each supplied list xpath. + * For example, if xpath '/parent/list' is provided, then list all elements in '/parent/list' will be deleted, + * e.g. /parent/list[@key='A'], /parent/list[@key='B']. + * This method will only delete whole lists by xpath; xpaths to list elements or other data nodes will be ignored. + * Non-existing xpaths will not result in an exception. + * @param anchorId the id of the anchor + * @param listXpaths xpaths of whole lists to remove + */ + void deleteListsByAnchorIdAndXpaths(int anchorId, Collection<String> listXpaths); } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java index 57dca568f2..0e4d359da5 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentNativeRepositoryImpl.java @@ -20,13 +20,12 @@ package org.onap.cps.spi.repository; -import java.sql.PreparedStatement; +import java.util.Collection; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; -import org.hibernate.Session; -import org.springframework.stereotype.Repository; +import lombok.RequiredArgsConstructor; -@Repository +@RequiredArgsConstructor public class FragmentNativeRepositoryImpl implements FragmentNativeRepository { private static final String DROP_FRAGMENT_CONSTRAINT @@ -34,28 +33,62 @@ public class FragmentNativeRepositoryImpl implements FragmentNativeRepository { private static final String ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE = "ALTER TABLE fragment ADD CONSTRAINT fragment_parent_id_fkey FOREIGN KEY (parent_id) " + "REFERENCES fragment (id) ON DELETE CASCADE;"; - private static final String DELETE_FRAGMENT = "DELETE FROM fragment WHERE id =?;"; private static final String ADD_ORIGINAL_FRAGMENT_CONSTRAINT = "ALTER TABLE fragment ADD CONSTRAINT fragment_parent_id_fkey FOREIGN KEY (parent_id) " + "REFERENCES fragment (id) ON DELETE NO ACTION;"; @PersistenceContext - private EntityManager entityManager; + private final EntityManager entityManager; + + private final TempTableCreator tempTableCreator; @Override public void deleteFragmentEntity(final long fragmentEntityId) { - final Session session = entityManager.unwrap(Session.class); - session.doWork(connection -> { - try (PreparedStatement preparedStatement = connection.prepareStatement( + entityManager.createNativeQuery( + DROP_FRAGMENT_CONSTRAINT + + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE + + "DELETE FROM fragment WHERE id = ?;" + + DROP_FRAGMENT_CONSTRAINT + + ADD_ORIGINAL_FRAGMENT_CONSTRAINT) + .setParameter(1, fragmentEntityId) + .executeUpdate(); + } + + @Override + // Accept security hotspot as temporary table name in SQL query is created internally, not from user input. + @SuppressWarnings("squid:S2077") + public void deleteByAnchorIdAndXpaths(final int anchorId, final Collection<String> xpaths) { + if (!xpaths.isEmpty()) { + final String tempTableName = tempTableCreator.createTemporaryTable("xpathsToDelete", xpaths, "xpath"); + entityManager.createNativeQuery( DROP_FRAGMENT_CONSTRAINT - + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE - + DELETE_FRAGMENT - + DROP_FRAGMENT_CONSTRAINT - + ADD_ORIGINAL_FRAGMENT_CONSTRAINT)) { - preparedStatement.setLong(1, fragmentEntityId); - preparedStatement.executeUpdate(); - } - }); + + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE + + "DELETE FROM fragment f USING " + tempTableName + " t" + + " WHERE f.anchor_id = :anchorId AND f.xpath = t.xpath;" + + DROP_FRAGMENT_CONSTRAINT + + ADD_ORIGINAL_FRAGMENT_CONSTRAINT) + .setParameter("anchorId", anchorId) + .executeUpdate(); + } } -} + @Override + // Accept security hotspot as temporary table name in SQL query is created internally, not from user input. + @SuppressWarnings("squid:S2077") + public void deleteListsByAnchorIdAndXpaths(final int anchorId, final Collection<String> xpaths) { + if (!xpaths.isEmpty()) { + final String tempTableName = tempTableCreator.createTemporaryTable("xpathsToDelete", xpaths, "xpath"); + entityManager.createNativeQuery( + DROP_FRAGMENT_CONSTRAINT + + ADD_FRAGMENT_CONSTRAINT_WITH_CASCADE + + "DELETE FROM fragment f USING " + tempTableName + " t" + + " WHERE f.anchor_id = :anchorId AND f.xpath LIKE CONCAT(t.xpath, :xpathListPattern);" + + DROP_FRAGMENT_CONSTRAINT + + ADD_ORIGINAL_FRAGMENT_CONSTRAINT) + .setParameter("anchorId", anchorId) + .setParameter("xpathListPattern", "[%%") + .executeUpdate(); + } + } + +} diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java index 4b42b2da8d..8bdb7d985b 100755 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepository.java @@ -1,6 +1,6 @@ /*
* ============LICENSE_START=======================================================
- * Copyright (C) 2021-2022 Nordix Foundation.
+ * Copyright (C) 2021-2023 Nordix Foundation.
* Modifications Copyright (C) 2020-2021 Bell Canada.
* Modifications Copyright (C) 2020-2021 Pantheon.tech.
* ================================================================================
@@ -40,7 +40,7 @@ import org.springframework.stereotype.Repository; @Repository
public interface FragmentRepository extends JpaRepository<FragmentEntity, Long>, FragmentRepositoryCpsPathQuery,
- FragmentRepositoryMultiPathQuery {
+ FragmentRepositoryMultiPathQuery, FragmentNativeRepository {
Optional<FragmentEntity> findByDataspaceAndAnchorAndXpath(@NonNull DataspaceEntity dataspaceEntity,
@NonNull AnchorEntity anchorEntity,
diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java index 8c357bbb31..151fe97b34 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/FragmentRepositoryMultiPathQueryImpl.java @@ -20,28 +20,24 @@ package org.onap.cps.spi.repository; - -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; import javax.transaction.Transactional; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.onap.cps.spi.entities.FragmentEntity; - @Slf4j -@AllArgsConstructor +@RequiredArgsConstructor public class FragmentRepositoryMultiPathQueryImpl implements FragmentRepositoryMultiPathQuery { @PersistenceContext - private EntityManager entityManager; + private final EntityManager entityManager; - private TempTableCreator tempTableCreator; + private final TempTableCreator tempTableCreator; @Override @Transactional @@ -50,24 +46,13 @@ public class FragmentRepositoryMultiPathQueryImpl implements FragmentRepositoryM if (cpsPathQueryList.isEmpty()) { return Collections.emptyList(); } - final Collection<List<String>> sqlData = new HashSet<>(cpsPathQueryList.size()); - for (final String query : cpsPathQueryList) { - final List<String> row = new ArrayList<>(1); - row.add(query); - sqlData.add(row); - } - final String tempTableName = tempTableCreator.createTemporaryTable( - "xpathTemporaryTable", sqlData, "xpath"); - return selectMatchingFragments(anchorId, tempTableName); - } - - private List<FragmentEntity> selectMatchingFragments(final Integer anchorId, final String tempTableName) { + "xpathTemporaryTable", cpsPathQueryList, "xpath"); final String sql = String.format( "SELECT * FROM FRAGMENT WHERE anchor_id = %d AND xpath IN (select xpath FROM %s);", anchorId, tempTableName); final List<FragmentEntity> fragmentEntities = entityManager.createNativeQuery(sql, FragmentEntity.class) - .getResultList(); + .getResultList(); log.debug("Fetched {} fragment entities by anchor and cps path.", fragmentEntities.size()); return fragmentEntities; } diff --git a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java index d713746e45..338b0b1c64 100644 --- a/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java +++ b/cps-ri/src/main/java/org/onap/cps/spi/repository/TempTableCreator.java @@ -1,6 +1,6 @@ /*- * ============LICENSE_START======================================================= - * Copyright (C) 2022 Nordix Foundation. + * Copyright (C) 2022-2023 Nordix Foundation. * ================================================================================ * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,10 @@ package org.onap.cps.spi.repository; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -63,6 +65,24 @@ public class TempTableCreator { return tempTableName; } + /** + * Create a uniquely named temporary table with a single column. + * + * @param prefix prefix for the table name (so you can recognize it) + * @param sqlData data to insert (strings only); each entry is a single row of data + * @param columnName column name + * @return a unique temporary table name with given prefix + */ + public String createTemporaryTable(final String prefix, + final Collection<String> sqlData, + final String columnName) { + final Collection<List<String>> tableData = new ArrayList<>(sqlData.size()); + for (final String entry : sqlData) { + tableData.add(Collections.singletonList(entry)); + } + return createTemporaryTable(prefix, tableData, columnName); + } + private static void defineColumns(final StringBuilder sqlStringBuilder, final String[] columnNames) { sqlStringBuilder.append('('); final Iterator<String> it = Arrays.stream(columnNames).iterator(); diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy index 5f48469c01..e4c552978d 100755 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy @@ -555,9 +555,10 @@ class CpsDataPersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase { def 'Delete list scenario: #scenario.'() { when: 'deleting list is executed for: #scenario.' objectUnderTest.deleteListDataNode(DATASPACE_NAME, ANCHOR_NAME3, targetXpaths) - then: 'only the expected children remain' + and: 'remaining children are fetched' def parentFragment = fragmentRepository.getById(parentFragmentId) def remainingChildXpaths = parentFragment.childFragments.collect { it.xpath } + then: 'only the expected children remain' assert remainingChildXpaths.size() == expectedRemainingChildXpaths.size() assert remainingChildXpaths.containsAll(expectedRemainingChildXpaths) where: 'following parameters were used' @@ -569,6 +570,29 @@ class CpsDataPersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase { } @Sql([CLEAR_DATA, SET_DATA]) + def 'Delete multiple data nodes using scenario: #scenario.'() { + when: 'deleting nodes is executed for: #scenario.' + objectUnderTest.deleteDataNodes(DATASPACE_NAME, ANCHOR_NAME3, targetXpaths) + and: 'remaining children are fetched' + def parentFragment = fragmentRepository.getById(LIST_DATA_NODE_PARENT203_FRAGMENT_ID) + def remainingChildXpaths = parentFragment.childFragments.collect { it.xpath } + then: 'only the expected children remain' + assert remainingChildXpaths.size() == expectedRemainingChildXpaths.size() + assert remainingChildXpaths.containsAll(expectedRemainingChildXpaths) + where: 'following parameters were used' + scenario | targetXpaths || expectedRemainingChildXpaths + 'delete nothing' | [] || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"] + 'datanode' | ['/parent-203/child-203'] || ["/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"] + '1 list element' | ['/parent-203/child-204[@key="A"]'] || ["/parent-203/child-203", "/parent-203/child-204[@key='B']"] + '2 list elements' | ['/parent-203/child-204[@key="A"]', '/parent-203/child-204[@key="B"]'] || ["/parent-203/child-203"] + 'whole list' | ['/parent-203/child-204'] || ['/parent-203/child-203'] + 'list and element in same list' | ['/parent-203/child-204', '/parent-203/child-204[@key="A"]'] || ['/parent-203/child-203'] + 'list element under list element' | ['/parent-203/child-204[@key="B"]/grand-child-204[@key2="Y"]'] || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"] + 'valid but non-existing xpath' | ['/non-existing', '/parent-203/child-204'] || ['/parent-203/child-203'] + 'invalid xpath' | ['INVALID XPATH', '/parent-203/child-204'] || ['/parent-203/child-203'] + } + + @Sql([CLEAR_DATA, SET_DATA]) def 'Delete data nodes with "/"-token in list key value: #scenario. (CPS-1409)'() { given: 'a data nodes with list-element child with "/" in index value (and grandchild)' def grandChild = new DataNodeBuilder().withXpath(deleteTestGrandChildXPath).build() diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy index 5dab87eec4..5cabc85b36 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy @@ -32,7 +32,6 @@ import org.onap.cps.spi.model.DataNode import org.onap.cps.spi.model.DataNodeBuilder import org.onap.cps.spi.repository.AnchorRepository import org.onap.cps.spi.repository.DataspaceRepository -import org.onap.cps.spi.repository.FragmentNativeRepository import org.onap.cps.spi.repository.FragmentRepository import org.onap.cps.spi.utils.SessionManager import org.onap.cps.utils.JsonObjectMapper @@ -46,10 +45,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def mockFragmentRepository = Mock(FragmentRepository) def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper()) def mockSessionManager = Mock(SessionManager) - def stubFragmentNativeRepository = Stub(FragmentNativeRepository) def objectUnderTest = Spy(new CpsDataPersistenceServiceImpl(mockDataspaceRepository, mockAnchorRepository, - mockFragmentRepository, jsonObjectMapper, mockSessionManager, stubFragmentNativeRepository)) + mockFragmentRepository, jsonObjectMapper, mockSessionManager)) def 'Storing data nodes individually when batch operation fails'(){ given: 'two data nodes and supporting repository mock behavior' diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy index 91da53d2ea..3b9338ce41 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy @@ -25,52 +25,57 @@ import org.onap.cps.spi.impl.CpsPersistencePerfSpecBase import org.springframework.beans.factory.annotation.Autowired import org.springframework.test.context.jdbc.Sql import org.springframework.util.StopWatch -import spock.lang.Shared - -import java.util.concurrent.TimeUnit class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase { @Autowired CpsDataPersistenceService objectUnderTest - static def NUMBER_OF_CHILDREN = 100 - static def NUMBER_OF_GRAND_CHILDREN = 50 - static def NUMBER_OF_LISTS = 100 - static def NUMBER_OF_LIST_ELEMENTS = 50 - static def ALLOWED_SETUP_TIME_MS = TimeUnit.SECONDS.toMillis(10) - def stopWatch = new StopWatch() @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() { when: 'a node with a large number of descendants is created' stopWatch.start() - createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false) + createLineage(objectUnderTest, 150, 50, false) stopWatch.stop() def setupDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' - recordAndAssertPerformance('Setup',ALLOWED_SETUP_TIME_MS, setupDurationInMillis) + then: 'setup duration is under 10 seconds' + recordAndAssertPerformance('Setup', 10_000, setupDurationInMillis) } def 'Delete 5 children with grandchildren'() { when: 'child nodes are deleted' stopWatch.start() (1..5).each { - def childPath = "${PERF_TEST_PARENT}/perf-test-child-${it}".toString(); + def childPath = "${PERF_TEST_PARENT}/perf-test-child-${it}".toString() objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, childPath) } stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 300 milliseconds' - recordAndAssertPerformance('Delete 5 children', 300, deleteDurationInMillis) + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Delete 5 children', 350, deleteDurationInMillis) + } + + def 'Batch delete 100 children with grandchildren'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = (6..105).collect { + "${PERF_TEST_PARENT}/perf-test-child-${it}".toString() + } + when: 'child nodes are deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 100 children', 350, deleteDurationInMillis) } def 'Delete 50 grandchildren (that have no descendants)'() { when: 'target nodes are deleted' stopWatch.start() (1..50).each { - def grandchildPath = "${PERF_TEST_PARENT}/perf-test-child-6/perf-test-grand-child-${it}".toString(); + def grandchildPath = "${PERF_TEST_PARENT}/perf-test-child-106/perf-test-grand-child-${it}".toString() objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, grandchildPath) } stopWatch.stop() @@ -79,78 +84,145 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase recordAndAssertPerformance('Delete 50 grandchildren', 350, deleteDurationInMillis) } - def 'Delete 1 large data node with many descendants'() { - when: 'parent node is deleted' + def 'Batch delete 500 grandchildren (that have no descendants)'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = [] + for (int childIndex = 0; childIndex < 10; childIndex++) { + xpathsToDelete.addAll((1..50).collect { + "${PERF_TEST_PARENT}/perf-test-child-${107+childIndex}/perf-test-grand-child-${it}".toString() + }) + } + when: 'target nodes are deleted' stopWatch.start() - objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, PERF_TEST_PARENT) + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 250 milliseconds' - recordAndAssertPerformance('Delete one large node', 250, deleteDurationInMillis) + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 500 grandchildren', 350, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Create a node with many list elements (please note, subsequent tests depend on this running first).'() { - given: 'a node with a large number of descendants is created' + when: 'a node with a large number of lists is created' stopWatch.start() - createLineage(objectUnderTest, NUMBER_OF_LISTS, NUMBER_OF_LIST_ELEMENTS, true) + createLineage(objectUnderTest, 150, 50, true) stopWatch.stop() def setupDurationInMillis = stopWatch.getTotalTimeMillis() - and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' - recordAndAssertPerformance('Create node with many list elements', ALLOWED_SETUP_TIME_MS, setupDurationInMillis) + then: 'setup duration is under 10 seconds' + recordAndAssertPerformance('Setup lists', 10_000, setupDurationInMillis) } - def 'Delete 5 whole lists with many elements'() { - when: 'list nodes are deleted' + def 'Delete 5 whole lists'() { + when: 'lists are deleted' stopWatch.start() (1..5).each { - def childPath = "${PERF_TEST_PARENT}/perf-test-list-${it}".toString(); + def childPath = "${PERF_TEST_PARENT}/perf-test-list-${it}".toString() objectUnderTest.deleteListDataNode(PERF_DATASPACE, PERF_ANCHOR, childPath) } stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 1000 milliseconds' + then: 'delete duration is under 1500 milliseconds' recordAndAssertPerformance('Delete 5 whole lists', 1500, deleteDurationInMillis) } - def 'Delete 10 list elements with keys'() { + def 'Batch delete 100 whole lists'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = (6..105).collect { + "${PERF_TEST_PARENT}/perf-test-list-${it}".toString() + } + when: 'lists are deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 100 whole lists', 350, deleteDurationInMillis) + } + + def 'Delete 10 list elements'() { when: 'list elements are deleted' stopWatch.start() (1..10).each { - def key = it.toString() - def grandchildPath = "${PERF_TEST_PARENT}/perf-test-list-6[@key='${key}']" + def grandchildPath = "${PERF_TEST_PARENT}/perf-test-list-106[@key='${it}']".toString() objectUnderTest.deleteListDataNode(PERF_DATASPACE, PERF_ANCHOR, grandchildPath) } stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 1200 milliseconds' - recordAndAssertPerformance('Delete 10 lists elements', 1500, deleteDurationInMillis) + then: 'delete duration is under 750 milliseconds' + recordAndAssertPerformance('Delete 10 lists elements', 750, deleteDurationInMillis) + } + + def 'Batch delete 500 list elements'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = [] + for (int childIndex = 0; childIndex < 10; childIndex++) { + xpathsToDelete.addAll((1..50).collect { + "${PERF_TEST_PARENT}/perf-test-list-${107+childIndex}[@key='${it}']".toString() + }) + } + when: 'list elements are deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 500 lists elements', 350, deleteDurationInMillis) + } + + @Sql([CLEAR_DATA, PERF_TEST_DATA]) + def 'Delete 1 large data node'() { + given: 'a node with a large number of descendants is created' + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) + when: 'parent node is deleted' + stopWatch.start() + objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, PERF_TEST_PARENT) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Delete one large node', 300, deleteDurationInMillis) + } + + @Sql([CLEAR_DATA, PERF_TEST_DATA]) + def 'Batch delete 1 large data node'() { + given: 'a node with a large number of descendants is created' + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) + when: 'parent node is batch deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, [PERF_TEST_PARENT]) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Batch delete one large node', 300, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Delete root node with many descendants'() { given: 'a node with a large number of descendants is created' - createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false) + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) when: 'root node is deleted' stopWatch.start() objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, '/') stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 250 milliseconds' - recordAndAssertPerformance('Delete root node', 250, deleteDurationInMillis) + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Delete root node', 300, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) - def 'Delete data nodes for an anchor'() { + def 'Delete data nodes for an anchor'() {212 given: 'a node with a large number of descendants is created' - createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false) + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) when: 'data nodes are deleted' stopWatch.start() objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR) stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 250 milliseconds' - recordAndAssertPerformance('Delete data nodes for anchor', 250, deleteDurationInMillis) + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Delete data nodes for anchor', 300, deleteDurationInMillis) } } diff --git a/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java b/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java index 6332f09109..174d71f64d 100644 --- a/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java +++ b/cps-service/src/main/java/org/onap/cps/api/CpsDataService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2022 Nordix Foundation + * Copyright (C) 2020-2023 Nordix Foundation * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2021-2022 Bell Canada * Modifications Copyright (C) 2022 Deutsche Telekom AG @@ -199,19 +199,30 @@ public interface CpsDataService { /** * Deletes data node for given anchor and dataspace. * - * @param dataspaceName dataspace name - * @param anchorName anchor name - * @param dataNodeXpath data node xpath + * @param dataspaceName dataspace name + * @param anchorName anchor name + * @param dataNodeXpath data node xpath * @param observedTimestamp observed timestamp */ void deleteDataNode(String dataspaceName, String anchorName, String dataNodeXpath, OffsetDateTime observedTimestamp); /** + * Deletes multiple data nodes for given anchor and dataspace. + * + * @param dataspaceName dataspace name + * @param anchorName anchor name + * @param dataNodeXpaths data node xpaths + * @param observedTimestamp observed timestamp + */ + void deleteDataNodes(String dataspaceName, String anchorName, Collection<String> dataNodeXpaths, + OffsetDateTime observedTimestamp); + + /** * Deletes all data nodes for a given anchor in a dataspace. * * @param dataspaceName dataspace name - * @param anchorName anchor name + * @param anchorName anchor name * @param observedTimestamp observed timestamp */ void deleteDataNodes(String dataspaceName, String anchorName, OffsetDateTime observedTimestamp); diff --git a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java b/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java index 53fab2916f..06a0845385 100755 --- a/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java +++ b/cps-service/src/main/java/org/onap/cps/api/impl/CpsDataServiceImpl.java @@ -264,7 +264,18 @@ public class CpsDataServiceImpl implements CpsDataService { @Timed(value = "cps.data.service.datanode.batch.delete", description = "Time taken to delete a batch of datanodes") public void deleteDataNodes(final String dataspaceName, final String anchorName, - final OffsetDateTime observedTimestamp) { + final Collection<String> dataNodeXpaths, final OffsetDateTime observedTimestamp) { + cpsValidator.validateNameCharacters(dataspaceName, anchorName); + cpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName, dataNodeXpaths); + dataNodeXpaths.forEach(dataNodeXpath -> + processDataUpdatedEventAsync(dataspaceName, anchorName, dataNodeXpath, DELETE, observedTimestamp)); + } + + @Override + @Timed(value = "cps.data.service.datanode.all.delete", + description = "Time taken to delete all datanodes") + public void deleteDataNodes(final String dataspaceName, final String anchorName, + final OffsetDateTime observedTimestamp) { cpsValidator.validateNameCharacters(dataspaceName, anchorName); processDataUpdatedEventAsync(dataspaceName, anchorName, ROOT_NODE_XPATH, DELETE, observedTimestamp); cpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName); diff --git a/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java b/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java index 0989ccae2d..3e0b4475e8 100644 --- a/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java +++ b/cps-service/src/main/java/org/onap/cps/spi/CpsDataPersistenceService.java @@ -1,6 +1,6 @@ /* * ============LICENSE_START======================================================= - * Copyright (C) 2020-2022 Nordix Foundation. + * Copyright (C) 2020-2023 Nordix Foundation. * Modifications Copyright (C) 2021 Pantheon.tech * Modifications Copyright (C) 2022 Bell Canada * Modifications Copyright (C) 2022 TechMahindra Ltd. @@ -174,6 +174,15 @@ public interface CpsDataPersistenceService { void deleteDataNode(String dataspaceName, String anchorName, String targetXpath); /** + * Deletes multiple dataNode, yang container or yang list or yang list element. + * + * @param dataspaceName dataspace name + * @param anchorName anchor name + * @param targetXpaths xpaths of nodes to delete + */ + void deleteDataNodes(String dataspaceName, String anchorName, Collection<String> targetXpaths); + + /** * Deletes all dataNodes in a given anchor. * * @param dataspaceName dataspace name @@ -182,7 +191,7 @@ public interface CpsDataPersistenceService { void deleteDataNodes(String dataspaceName, String anchorName); /** - * Deletes existing a single list element or the whole list. + * Deletes a single existing list element or the whole list. * * @param dataspaceName dataspace name * @param anchorName anchor name diff --git a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy index 01dc0bde42..8bbf4e5715 100644 --- a/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy +++ b/cps-service/src/test/groovy/org/onap/cps/api/impl/CpsDataServiceImplSpec.groovy @@ -325,6 +325,19 @@ class CpsDataServiceImplSpec extends Specification { 1 * mockNotificationService.processDataUpdatedEvent(dataspaceName, anchorName, '/test-tree/branch', Operation.DELETE, observedTimestamp) } + def 'Delete multiple list elements under existing node.'() { + given: 'schema set for given anchor and dataspace references test-tree model' + setupSchemaSetMocks('test-tree.yang') + when: 'delete multiple list data method is invoked with list element json data' + objectUnderTest.deleteDataNodes(dataspaceName, anchorName, ['/test-tree/branch[@name="A"]', '/test-tree/branch[@name="B"]'], observedTimestamp) + then: 'the persistence service method is invoked with correct parameters' + 1 * mockCpsDataPersistenceService.deleteDataNodes(dataspaceName, anchorName, ['/test-tree/branch[@name="A"]', '/test-tree/branch[@name="B"]']) + and: 'the CpsValidator is called on the dataspaceName and AnchorName' + 1 * mockCpsValidator.validateNameCharacters(dataspaceName, anchorName) + and: 'two data updated events are sent to notification service' + 2 * mockNotificationService.processDataUpdatedEvent(dataspaceName, anchorName, _, Operation.DELETE, observedTimestamp) + } + def 'Delete data node under anchor and dataspace.'() { given: 'schema set for given anchor and dataspace references test tree model' setupSchemaSetMocks('test-tree.yang') diff --git a/csit/plans/cps/setup.sh b/csit/plans/cps/setup.sh index 43575f9446..c715da142f 100755 --- a/csit/plans/cps/setup.sh +++ b/csit/plans/cps/setup.sh @@ -57,9 +57,7 @@ source $WORKSPACE/plans/cps/test.properties export $(cut -d= -f1 $WORKSPACE/plans/cps/test.properties) ###################### setup cps-ncmp ############################ -mkdir -p $WORKSPACE/archives/dc-cps -cp $WORKSPACE/../docker-compose/*.yml $WORKSPACE/archives/dc-cps -cd $WORKSPACE/archives/dc-cps +cd $CPS_HOME/docker-compose curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-`uname -s`-`uname -m` > docker-compose chmod +x docker-compose diff --git a/csit/plans/cps/test.properties b/csit/plans/cps/test.properties index 9e22362121..6a715f75df 100644 --- a/csit/plans/cps/test.properties +++ b/csit/plans/cps/test.properties @@ -27,3 +27,5 @@ DMI_VERSION=1.3.0-SNAPSHOT-latest ADVISED_MODULES_SYNC_SLEEP_TIME_MS=2000 CMHANDLE_DATA_SYNC_SLEEP_TIME_MS=2000 + +CPS_HOME=$CPS_HOME
\ No newline at end of file diff --git a/csit/run-project-csit.sh b/csit/run-project-csit.sh index a231c5e160..fcb3c925c1 100755 --- a/csit/run-project-csit.sh +++ b/csit/run-project-csit.sh @@ -21,6 +21,7 @@ # $1 test options (passed on to run-csit.sh as such) export TESTOPTIONS=${1} +export CPS_HOME=$(git rev-parse --show-toplevel) export WORKSPACE=$(git rev-parse --show-toplevel)/csit rm -rf ${WORKSPACE}/archives diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 7d6379ac2f..dd7749ac58 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -110,13 +110,12 @@ services: restart: unless-stopped init-db: - build: ../../../docker-compose/initfile + build: ${CPS_HOME:-.}/docker-compose/initfile environment: CPS_CORE_PORT: ${CPS_CORE_PORT:-8883} CPS_CORE_MANAGEMENT_PORT: ${CPS_CORE_MANAGEMENT_PORT:-8887} volumes: - - ./initfile:/initfile - - ../../../cps-ncmp-service/src/main/resources/model/:/model + - ${CPS_HOME:-.}/cps-ncmp-service/src/main/resources/model/:/model depends_on: cps-and-ncmp: condition: service_started diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy index 0e04d62dd8..0673f7eb43 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/TestConfig.groovy @@ -29,7 +29,6 @@ import org.onap.cps.spi.impl.CpsDataPersistenceServiceImpl import org.onap.cps.spi.impl.CpsModulePersistenceServiceImpl import org.onap.cps.spi.repository.AnchorRepository import org.onap.cps.spi.repository.DataspaceRepository -import org.onap.cps.spi.repository.FragmentNativeRepository import org.onap.cps.spi.repository.FragmentRepository import org.onap.cps.spi.repository.ModuleReferenceRepository import org.onap.cps.spi.repository.SchemaSetRepository @@ -72,10 +71,6 @@ class TestConfig extends Specification{ @Autowired @Lazy - FragmentNativeRepository fragmentNativeRepository - - @Autowired - @Lazy JsonObjectMapper jsonObjectMapper @Autowired @@ -89,7 +84,7 @@ class TestConfig extends Specification{ @Bean CpsDataPersistenceService cpsDataPersistenceService() { - return (CpsDataPersistenceService) new CpsDataPersistenceServiceImpl(dataspaceRepository, anchorRepository, fragmentRepository, jsonObjectMapper, stubbedSessionManager, fragmentNativeRepository) + return (CpsDataPersistenceService) new CpsDataPersistenceServiceImpl(dataspaceRepository, anchorRepository, fragmentRepository, jsonObjectMapper, stubbedSessionManager) } @Bean |