From 230b1119dec71e301ba462246c3fc53d0fc0281a Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Tue, 24 Jan 2023 11:23:02 +0000 Subject: Create plural version of deleteDataNode - Add method to CpsDataService to batch delete data nodes and lists - Use native queries to batch delete fragment entities by xpaths, for data nodes and lists - Add performance tests for batch delete - Refactor FragmentNativeRepository - Add single-column version of createTemporaryTable - Renamed metric cps.data.service.datanode.batch.delete to cps.data.service.datanode.all.delete Issue-ID: CPS-1438 Signed-off-by: danielhanrahan Change-Id: I1851f9c7ef0b1be4bd421b3352d9697a2dd23f79 --- ...CpsDataPersistenceServiceIntegrationSpec.groovy | 26 +++- .../spi/impl/CpsDataPersistenceServiceSpec.groovy | 4 +- .../CpsDataPersistenceServiceDeletePerfTest.groovy | 154 +++++++++++++++------ 3 files changed, 139 insertions(+), 45 deletions(-) (limited to 'cps-ri/src/test') diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy index 5f48469c0..e4c552978 100755 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceIntegrationSpec.groovy @@ -555,9 +555,10 @@ class CpsDataPersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase { def 'Delete list scenario: #scenario.'() { when: 'deleting list is executed for: #scenario.' objectUnderTest.deleteListDataNode(DATASPACE_NAME, ANCHOR_NAME3, targetXpaths) - then: 'only the expected children remain' + and: 'remaining children are fetched' def parentFragment = fragmentRepository.getById(parentFragmentId) def remainingChildXpaths = parentFragment.childFragments.collect { it.xpath } + then: 'only the expected children remain' assert remainingChildXpaths.size() == expectedRemainingChildXpaths.size() assert remainingChildXpaths.containsAll(expectedRemainingChildXpaths) where: 'following parameters were used' @@ -568,6 +569,29 @@ class CpsDataPersistenceServiceIntegrationSpec extends CpsPersistenceSpecBase { 'list element under list element' | '/parent-203/child-204[@key="B"]/grand-child-204[@key2="Y"]' | LIST_DATA_NODE_PARENT203_FRAGMENT_ID || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"] } + @Sql([CLEAR_DATA, SET_DATA]) + def 'Delete multiple data nodes using scenario: #scenario.'() { + when: 'deleting nodes is executed for: #scenario.' + objectUnderTest.deleteDataNodes(DATASPACE_NAME, ANCHOR_NAME3, targetXpaths) + and: 'remaining children are fetched' + def parentFragment = fragmentRepository.getById(LIST_DATA_NODE_PARENT203_FRAGMENT_ID) + def remainingChildXpaths = parentFragment.childFragments.collect { it.xpath } + then: 'only the expected children remain' + assert remainingChildXpaths.size() == expectedRemainingChildXpaths.size() + assert remainingChildXpaths.containsAll(expectedRemainingChildXpaths) + where: 'following parameters were used' + scenario | targetXpaths || expectedRemainingChildXpaths + 'delete nothing' | [] || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"] + 'datanode' | ['/parent-203/child-203'] || ["/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"] + '1 list element' | ['/parent-203/child-204[@key="A"]'] || ["/parent-203/child-203", "/parent-203/child-204[@key='B']"] + '2 list elements' | ['/parent-203/child-204[@key="A"]', '/parent-203/child-204[@key="B"]'] || ["/parent-203/child-203"] + 'whole list' | ['/parent-203/child-204'] || ['/parent-203/child-203'] + 'list and element in same list' | ['/parent-203/child-204', '/parent-203/child-204[@key="A"]'] || ['/parent-203/child-203'] + 'list element under list element' | ['/parent-203/child-204[@key="B"]/grand-child-204[@key2="Y"]'] || ["/parent-203/child-203", "/parent-203/child-204[@key='A']", "/parent-203/child-204[@key='B']"] + 'valid but non-existing xpath' | ['/non-existing', '/parent-203/child-204'] || ['/parent-203/child-203'] + 'invalid xpath' | ['INVALID XPATH', '/parent-203/child-204'] || ['/parent-203/child-203'] + } + @Sql([CLEAR_DATA, SET_DATA]) def 'Delete data nodes with "/"-token in list key value: #scenario. (CPS-1409)'() { given: 'a data nodes with list-element child with "/" in index value (and grandchild)' diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy index 5dab87eec..5cabc85b3 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy @@ -32,7 +32,6 @@ import org.onap.cps.spi.model.DataNode import org.onap.cps.spi.model.DataNodeBuilder import org.onap.cps.spi.repository.AnchorRepository import org.onap.cps.spi.repository.DataspaceRepository -import org.onap.cps.spi.repository.FragmentNativeRepository import org.onap.cps.spi.repository.FragmentRepository import org.onap.cps.spi.utils.SessionManager import org.onap.cps.utils.JsonObjectMapper @@ -46,10 +45,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def mockFragmentRepository = Mock(FragmentRepository) def jsonObjectMapper = new JsonObjectMapper(new ObjectMapper()) def mockSessionManager = Mock(SessionManager) - def stubFragmentNativeRepository = Stub(FragmentNativeRepository) def objectUnderTest = Spy(new CpsDataPersistenceServiceImpl(mockDataspaceRepository, mockAnchorRepository, - mockFragmentRepository, jsonObjectMapper, mockSessionManager, stubFragmentNativeRepository)) + mockFragmentRepository, jsonObjectMapper, mockSessionManager)) def 'Storing data nodes individually when batch operation fails'(){ given: 'two data nodes and supporting repository mock behavior' diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy index 91da53d2e..3b9338ce4 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/performance/CpsDataPersistenceServiceDeletePerfTest.groovy @@ -25,52 +25,57 @@ import org.onap.cps.spi.impl.CpsPersistencePerfSpecBase import org.springframework.beans.factory.annotation.Autowired import org.springframework.test.context.jdbc.Sql import org.springframework.util.StopWatch -import spock.lang.Shared - -import java.util.concurrent.TimeUnit class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase { @Autowired CpsDataPersistenceService objectUnderTest - static def NUMBER_OF_CHILDREN = 100 - static def NUMBER_OF_GRAND_CHILDREN = 50 - static def NUMBER_OF_LISTS = 100 - static def NUMBER_OF_LIST_ELEMENTS = 50 - static def ALLOWED_SETUP_TIME_MS = TimeUnit.SECONDS.toMillis(10) - def stopWatch = new StopWatch() @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Create a node with many descendants (please note, subsequent tests depend on this running first).'() { when: 'a node with a large number of descendants is created' stopWatch.start() - createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false) + createLineage(objectUnderTest, 150, 50, false) stopWatch.stop() def setupDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' - recordAndAssertPerformance('Setup',ALLOWED_SETUP_TIME_MS, setupDurationInMillis) + then: 'setup duration is under 10 seconds' + recordAndAssertPerformance('Setup', 10_000, setupDurationInMillis) } def 'Delete 5 children with grandchildren'() { when: 'child nodes are deleted' stopWatch.start() (1..5).each { - def childPath = "${PERF_TEST_PARENT}/perf-test-child-${it}".toString(); + def childPath = "${PERF_TEST_PARENT}/perf-test-child-${it}".toString() objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, childPath) } stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 300 milliseconds' - recordAndAssertPerformance('Delete 5 children', 300, deleteDurationInMillis) + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Delete 5 children', 350, deleteDurationInMillis) + } + + def 'Batch delete 100 children with grandchildren'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = (6..105).collect { + "${PERF_TEST_PARENT}/perf-test-child-${it}".toString() + } + when: 'child nodes are deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 100 children', 350, deleteDurationInMillis) } def 'Delete 50 grandchildren (that have no descendants)'() { when: 'target nodes are deleted' stopWatch.start() (1..50).each { - def grandchildPath = "${PERF_TEST_PARENT}/perf-test-child-6/perf-test-grand-child-${it}".toString(); + def grandchildPath = "${PERF_TEST_PARENT}/perf-test-child-106/perf-test-grand-child-${it}".toString() objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, grandchildPath) } stopWatch.stop() @@ -79,78 +84,145 @@ class CpsDataPersistenceServiceDeletePerfTest extends CpsPersistencePerfSpecBase recordAndAssertPerformance('Delete 50 grandchildren', 350, deleteDurationInMillis) } - def 'Delete 1 large data node with many descendants'() { - when: 'parent node is deleted' + def 'Batch delete 500 grandchildren (that have no descendants)'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = [] + for (int childIndex = 0; childIndex < 10; childIndex++) { + xpathsToDelete.addAll((1..50).collect { + "${PERF_TEST_PARENT}/perf-test-child-${107+childIndex}/perf-test-grand-child-${it}".toString() + }) + } + when: 'target nodes are deleted' stopWatch.start() - objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, PERF_TEST_PARENT) + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 250 milliseconds' - recordAndAssertPerformance('Delete one large node', 250, deleteDurationInMillis) + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 500 grandchildren', 350, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Create a node with many list elements (please note, subsequent tests depend on this running first).'() { - given: 'a node with a large number of descendants is created' + when: 'a node with a large number of lists is created' stopWatch.start() - createLineage(objectUnderTest, NUMBER_OF_LISTS, NUMBER_OF_LIST_ELEMENTS, true) + createLineage(objectUnderTest, 150, 50, true) stopWatch.stop() def setupDurationInMillis = stopWatch.getTotalTimeMillis() - and: 'setup duration is under #ALLOWED_SETUP_TIME_MS milliseconds' - recordAndAssertPerformance('Create node with many list elements', ALLOWED_SETUP_TIME_MS, setupDurationInMillis) + then: 'setup duration is under 10 seconds' + recordAndAssertPerformance('Setup lists', 10_000, setupDurationInMillis) } - def 'Delete 5 whole lists with many elements'() { - when: 'list nodes are deleted' + def 'Delete 5 whole lists'() { + when: 'lists are deleted' stopWatch.start() (1..5).each { - def childPath = "${PERF_TEST_PARENT}/perf-test-list-${it}".toString(); + def childPath = "${PERF_TEST_PARENT}/perf-test-list-${it}".toString() objectUnderTest.deleteListDataNode(PERF_DATASPACE, PERF_ANCHOR, childPath) } stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 1000 milliseconds' + then: 'delete duration is under 1500 milliseconds' recordAndAssertPerformance('Delete 5 whole lists', 1500, deleteDurationInMillis) } - def 'Delete 10 list elements with keys'() { + def 'Batch delete 100 whole lists'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = (6..105).collect { + "${PERF_TEST_PARENT}/perf-test-list-${it}".toString() + } + when: 'lists are deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 100 whole lists', 350, deleteDurationInMillis) + } + + def 'Delete 10 list elements'() { when: 'list elements are deleted' stopWatch.start() (1..10).each { - def key = it.toString() - def grandchildPath = "${PERF_TEST_PARENT}/perf-test-list-6[@key='${key}']" + def grandchildPath = "${PERF_TEST_PARENT}/perf-test-list-106[@key='${it}']".toString() objectUnderTest.deleteListDataNode(PERF_DATASPACE, PERF_ANCHOR, grandchildPath) } stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 1200 milliseconds' - recordAndAssertPerformance('Delete 10 lists elements', 1500, deleteDurationInMillis) + then: 'delete duration is under 750 milliseconds' + recordAndAssertPerformance('Delete 10 lists elements', 750, deleteDurationInMillis) + } + + def 'Batch delete 500 list elements'() { + given: 'a list of xpaths to delete' + def xpathsToDelete = [] + for (int childIndex = 0; childIndex < 10; childIndex++) { + xpathsToDelete.addAll((1..50).collect { + "${PERF_TEST_PARENT}/perf-test-list-${107+childIndex}[@key='${it}']".toString() + }) + } + when: 'list elements are deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, xpathsToDelete) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 350 milliseconds' + recordAndAssertPerformance('Batch delete 500 lists elements', 350, deleteDurationInMillis) + } + + @Sql([CLEAR_DATA, PERF_TEST_DATA]) + def 'Delete 1 large data node'() { + given: 'a node with a large number of descendants is created' + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) + when: 'parent node is deleted' + stopWatch.start() + objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, PERF_TEST_PARENT) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Delete one large node', 300, deleteDurationInMillis) + } + + @Sql([CLEAR_DATA, PERF_TEST_DATA]) + def 'Batch delete 1 large data node'() { + given: 'a node with a large number of descendants is created' + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) + when: 'parent node is batch deleted' + stopWatch.start() + objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR, [PERF_TEST_PARENT]) + stopWatch.stop() + def deleteDurationInMillis = stopWatch.getTotalTimeMillis() + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Batch delete one large node', 300, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) def 'Delete root node with many descendants'() { given: 'a node with a large number of descendants is created' - createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false) + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) when: 'root node is deleted' stopWatch.start() objectUnderTest.deleteDataNode(PERF_DATASPACE, PERF_ANCHOR, '/') stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 250 milliseconds' - recordAndAssertPerformance('Delete root node', 250, deleteDurationInMillis) + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Delete root node', 300, deleteDurationInMillis) } @Sql([CLEAR_DATA, PERF_TEST_DATA]) - def 'Delete data nodes for an anchor'() { + def 'Delete data nodes for an anchor'() {212 given: 'a node with a large number of descendants is created' - createLineage(objectUnderTest, NUMBER_OF_CHILDREN, NUMBER_OF_GRAND_CHILDREN, false) + createLineage(objectUnderTest, 50, 50, false) + createLineage(objectUnderTest, 50, 50, true) when: 'data nodes are deleted' stopWatch.start() objectUnderTest.deleteDataNodes(PERF_DATASPACE, PERF_ANCHOR) stopWatch.stop() def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is under 250 milliseconds' - recordAndAssertPerformance('Delete data nodes for anchor', 250, deleteDurationInMillis) + then: 'delete duration is under 300 milliseconds' + recordAndAssertPerformance('Delete data nodes for anchor', 300, deleteDurationInMillis) } } -- cgit 1.2.3-korg