summaryrefslogtreecommitdiffstats
path: root/cps-ri/src/test/groovy
diff options
context:
space:
mode:
authordanielhanrahan <daniel.hanrahan@est.tech>2023-06-08 14:37:17 +0100
committerdanielhanrahan <daniel.hanrahan@est.tech>2023-06-14 14:29:06 +0100
commit9474e8f257f9a03c80c01cbf3729cd128a43847b (patch)
tree30c9f019e3e7df44d37ae8e4ffb695426b01ac2b /cps-ri/src/test/groovy
parent1368fd006373dd209a34274723fbda6ecb9d317f (diff)
Lower memory usage in FragmentRepository
Avoid using Spring Data "interface projection" in FragmentRepository. The use of FragmentExtract in FragmentRepository is causing an overhead of around 5 kilobytes per fragment, which is leading to abnormally high memory usage when queries return a large number of nodes. For example, around 250MB of additional memory is needlessly used when fetching 50,000 datanodes. - Remove FragmentExtract interface and FragmentEntityArranger class. - Add FragmentPrefetchRepository, using JdbcTemplate and RowMapper to fetch FragmentEntity descendants in a single SQL query. - Many CpsDataService operations have memory reductions: - queryDataNodes - getDataNodesForMultipleXpaths - updateDataNodesAndDescendants - updateNodeLeaves - and any NCMP methods using the above. Issue-ID: CPS-1716 Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech> Change-Id: Ic47a2c9eb34150ed76bd5ce452fe1c9aaf9b4c5c
Diffstat (limited to 'cps-ri/src/test/groovy')
-rw-r--r--cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy56
1 files changed, 23 insertions, 33 deletions
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy
index e8921b3ed0..cb554faee8 100644
--- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy
+++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy
@@ -26,7 +26,7 @@ import org.onap.cps.spi.FetchDescendantsOption
import org.onap.cps.spi.entities.AnchorEntity
import org.onap.cps.spi.entities.DataspaceEntity
import org.onap.cps.spi.entities.FragmentEntity
-import org.onap.cps.spi.entities.FragmentExtract
+
import org.onap.cps.spi.exceptions.ConcurrencyException
import org.onap.cps.spi.exceptions.DataValidationException
import org.onap.cps.spi.model.DataNode
@@ -55,6 +55,7 @@ class CpsDataPersistenceServiceSpec extends Specification {
def setup() {
mockAnchorRepository.getByDataspaceAndName(_, _) >> anchorEntity
+ mockFragmentRepository.prefetchDescendantsOfFragmentEntities(_, _) >> { fetchDescendantsOption, fragmentEntities -> fragmentEntities }
}
def 'Storing data nodes individually when batch operation fails'(){
@@ -93,20 +94,20 @@ class CpsDataPersistenceServiceSpec extends Specification {
def 'Batch update data node leaves and descendants: #scenario'(){
given: 'the fragment repository returns fragment entities related to the xpath inputs'
- mockFragmentRepository.findExtractsWithDescendants(_, [] as Set, _) >> []
- mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath'] as Set, _) >> [
- mockFragmentExtract(1, null, 123, '/test/xpath', "{\"id\":\"testId1\"}")
+ mockFragmentRepository.findByAnchorAndXpathIn(_, [] as Set) >> []
+ mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath'] as Set) >> [
+ new FragmentEntity(1, '/test/xpath', null, "{\"id\":\"testId\"}", anchorEntity, [] as Set)
]
- mockFragmentRepository.findExtractsWithDescendants(123, ['/test/xpath1', '/test/xpath2'] as Set, _) >> [
- mockFragmentExtract(1, null, 123, '/test/xpath1', "{\"id\":\"testId1\"}"),
- mockFragmentExtract(2, null, 123, '/test/xpath2', "{\"id\":\"testId1\"}")
+ mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath1', '/test/xpath2'] as Set) >> [
+ new FragmentEntity(1, '/test/xpath1', null, "{\"id\":\"testId1\"}", anchorEntity, [] as Set),
+ new FragmentEntity(2, '/test/xpath2', null, "{\"id\":\"testId2\"}", anchorEntity, [] as Set)
]
when: 'replace data node tree'
objectUnderTest.batchUpdateDataLeaves('dataspaceName', 'anchorName',
dataNodes.stream().collect(Collectors.toMap(DataNode::getXpath, DataNode::getLeaves)))
then: 'call fragment repository save all method'
1 * mockFragmentRepository.saveAll({fragmentEntities ->
- assert fragmentEntities as List == expectedFragmentEntities
+ assert fragmentEntities.sort() == expectedFragmentEntities.sort()
assert fragmentEntities.size() == expectedSize
})
where: 'the following Data Type is passed'
@@ -172,9 +173,9 @@ class CpsDataPersistenceServiceSpec extends Specification {
def 'Retrieving multiple data nodes.'() {
given: 'fragment repository returns a collection of fragments'
- mockFragmentRepository.findExtractsWithDescendants(123, ['/xpath1', '/xpath2'] as Set, _) >> [
- mockFragmentExtract(1, null, 123, '/xpath1', null),
- mockFragmentExtract(2, null, 123, '/xpath2', null)
+ mockFragmentRepository.findByAnchorAndXpathIn(anchorEntity, ['/xpath1', '/xpath2'] as Set) >> [
+ new FragmentEntity(1, '/xpath1', null, null, anchorEntity, [] as Set),
+ new FragmentEntity(2, '/xpath2', null, null, anchorEntity, [] as Set)
]
when: 'getting data nodes for 2 xpaths'
def result = objectUnderTest.getDataNodesForMultipleXpaths('some-dataspace', 'some-anchor', ['/xpath1', '/xpath2'], FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS)
@@ -207,9 +208,9 @@ class CpsDataPersistenceServiceSpec extends Specification {
def 'Replace data node and descendants: #scenario'(){
given: 'the fragment repository returns fragment entities related to the xpath inputs'
- mockFragmentRepository.findExtractsWithDescendants(_, [] as Set, _) >> []
- mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath'] as Set, _) >> [
- mockFragmentExtract(1, null, 123, '/test/xpath', null)
+ mockFragmentRepository.findByAnchorAndXpathIn(_, [] as Set) >> []
+ mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath'] as Set) >> [
+ new FragmentEntity(1, '/test/xpath', null, '{"id":"testId"}', anchorEntity, [] as Set)
]
when: 'replace data node tree'
objectUnderTest.updateDataNodesAndDescendants('dataspaceName', 'anchorName', dataNodes)
@@ -223,9 +224,9 @@ class CpsDataPersistenceServiceSpec extends Specification {
def 'Replace data nodes and descendants'() {
given: 'the fragment repository returns fragment entities related to the xpath inputs'
- mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath1', '/test/xpath2'] as Set, _) >> [
- mockFragmentExtract(1, null, 123, '/test/xpath1', null),
- mockFragmentExtract(2, null, 123, '/test/xpath2', null)
+ mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath1', '/test/xpath2'] as Set) >> [
+ new FragmentEntity(1, '/test/xpath1', null, null, anchorEntity, [] as Set),
+ new FragmentEntity(2, '/test/xpath2', null, null, anchorEntity, [] as Set)
]
and: 'some data nodes with descendants'
def dataNode1 = new DataNode(xpath: '/test/xpath1', leaves: ['id': 'testId1'], childDataNodes: [new DataNode(xpath: '/test/xpath1/child', leaves: ['id': 'childTestId1'])])
@@ -253,38 +254,27 @@ class CpsDataPersistenceServiceSpec extends Specification {
def createDataNodesAndMockRepositoryMethodSupportingThem(Map<String, String> xpathToScenarioMap) {
def dataNodes = []
- def fragmentExtracts = []
+ def fragmentEntities = []
def fragmentId = 1
xpathToScenarioMap.each {
def xpath = it.key
def scenario = it.value
def dataNode = new DataNodeBuilder().withXpath(xpath).build()
dataNodes.add(dataNode)
- def fragmentExtract = mockFragmentExtract(fragmentId, null, 123, xpath, null)
- fragmentExtracts.add(fragmentExtract)
def fragmentEntity = new FragmentEntity(id: fragmentId, anchor: anchorEntity, xpath: xpath, childFragments: [])
+ fragmentEntities.add(fragmentEntity)
if ('EXCEPTION' == scenario) {
mockFragmentRepository.save(fragmentEntity) >> { throw new StaleStateException("concurrent updates") }
}
fragmentId++
}
- mockFragmentRepository.findExtractsWithDescendants(_, xpathToScenarioMap.keySet(), _) >> fragmentExtracts
+ mockFragmentRepository.findByAnchorAndXpathIn(_, xpathToScenarioMap.keySet()) >> fragmentEntities
return dataNodes
}
def mockFragmentWithJson(json) {
- def fragmentExtract = mockFragmentExtract(456, null, 123, '/parent-01', json)
- mockFragmentRepository.findExtractsWithDescendants(123, ['/parent-01'] as Set, _) >> [fragmentExtract]
- }
-
- def mockFragmentExtract(id, parentId, anchorId, xpath, attributes) {
- def fragmentExtract = Mock(FragmentExtract)
- fragmentExtract.getId() >> id
- fragmentExtract.getParentId() >> parentId
- fragmentExtract.getAnchorId() >> anchorId
- fragmentExtract.getXpath() >> xpath
- fragmentExtract.getAttributes() >> attributes
- return fragmentExtract
+ def fragmentEntity = new FragmentEntity(456, '/parent-01', null, json, anchorEntity, [] as Set)
+ mockFragmentRepository.findByAnchorAndXpathIn(_, ['/parent-01'] as Set) >> [fragmentEntity]
}
}