diff options
author | 2023-06-08 14:37:17 +0100 | |
---|---|---|
committer | 2023-06-14 14:29:06 +0100 | |
commit | 9474e8f257f9a03c80c01cbf3729cd128a43847b (patch) | |
tree | 30c9f019e3e7df44d37ae8e4ffb695426b01ac2b /cps-ri/src/test | |
parent | 1368fd006373dd209a34274723fbda6ecb9d317f (diff) |
Lower memory usage in FragmentRepository
Avoid using Spring Data "interface projection" in FragmentRepository.
The use of FragmentExtract in FragmentRepository is causing an
overhead of around 5 kilobytes per fragment, which is leading to
abnormally high memory usage when queries return a large number of
nodes. For example, around 250MB of additional memory is needlessly
used when fetching 50,000 datanodes.
- Remove FragmentExtract interface and FragmentEntityArranger class.
- Add FragmentPrefetchRepository, using JdbcTemplate and RowMapper
to fetch FragmentEntity descendants in a single SQL query.
- Many CpsDataService operations have memory reductions:
- queryDataNodes
- getDataNodesForMultipleXpaths
- updateDataNodesAndDescendants
- updateNodeLeaves
- and any NCMP methods using the above.
Issue-ID: CPS-1716
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: Ic47a2c9eb34150ed76bd5ce452fe1c9aaf9b4c5c
Diffstat (limited to 'cps-ri/src/test')
-rw-r--r-- | cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy | 56 |
1 files changed, 23 insertions, 33 deletions
diff --git a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy index e8921b3ed..cb554faee 100644 --- a/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy +++ b/cps-ri/src/test/groovy/org/onap/cps/spi/impl/CpsDataPersistenceServiceSpec.groovy @@ -26,7 +26,7 @@ import org.onap.cps.spi.FetchDescendantsOption import org.onap.cps.spi.entities.AnchorEntity import org.onap.cps.spi.entities.DataspaceEntity import org.onap.cps.spi.entities.FragmentEntity -import org.onap.cps.spi.entities.FragmentExtract + import org.onap.cps.spi.exceptions.ConcurrencyException import org.onap.cps.spi.exceptions.DataValidationException import org.onap.cps.spi.model.DataNode @@ -55,6 +55,7 @@ class CpsDataPersistenceServiceSpec extends Specification { def setup() { mockAnchorRepository.getByDataspaceAndName(_, _) >> anchorEntity + mockFragmentRepository.prefetchDescendantsOfFragmentEntities(_, _) >> { fetchDescendantsOption, fragmentEntities -> fragmentEntities } } def 'Storing data nodes individually when batch operation fails'(){ @@ -93,20 +94,20 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Batch update data node leaves and descendants: #scenario'(){ given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findExtractsWithDescendants(_, [] as Set, _) >> [] - mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath', "{\"id\":\"testId1\"}") + mockFragmentRepository.findByAnchorAndXpathIn(_, [] as Set) >> [] + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath'] as Set) >> [ + new FragmentEntity(1, '/test/xpath', null, "{\"id\":\"testId\"}", anchorEntity, [] as Set) ] - mockFragmentRepository.findExtractsWithDescendants(123, ['/test/xpath1', '/test/xpath2'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath1', "{\"id\":\"testId1\"}"), - mockFragmentExtract(2, null, 123, '/test/xpath2', "{\"id\":\"testId1\"}") + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath1', '/test/xpath2'] as Set) >> [ + new FragmentEntity(1, '/test/xpath1', null, "{\"id\":\"testId1\"}", anchorEntity, [] as Set), + new FragmentEntity(2, '/test/xpath2', null, "{\"id\":\"testId2\"}", anchorEntity, [] as Set) ] when: 'replace data node tree' objectUnderTest.batchUpdateDataLeaves('dataspaceName', 'anchorName', dataNodes.stream().collect(Collectors.toMap(DataNode::getXpath, DataNode::getLeaves))) then: 'call fragment repository save all method' 1 * mockFragmentRepository.saveAll({fragmentEntities -> - assert fragmentEntities as List == expectedFragmentEntities + assert fragmentEntities.sort() == expectedFragmentEntities.sort() assert fragmentEntities.size() == expectedSize }) where: 'the following Data Type is passed' @@ -172,9 +173,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Retrieving multiple data nodes.'() { given: 'fragment repository returns a collection of fragments' - mockFragmentRepository.findExtractsWithDescendants(123, ['/xpath1', '/xpath2'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/xpath1', null), - mockFragmentExtract(2, null, 123, '/xpath2', null) + mockFragmentRepository.findByAnchorAndXpathIn(anchorEntity, ['/xpath1', '/xpath2'] as Set) >> [ + new FragmentEntity(1, '/xpath1', null, null, anchorEntity, [] as Set), + new FragmentEntity(2, '/xpath2', null, null, anchorEntity, [] as Set) ] when: 'getting data nodes for 2 xpaths' def result = objectUnderTest.getDataNodesForMultipleXpaths('some-dataspace', 'some-anchor', ['/xpath1', '/xpath2'], FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS) @@ -207,9 +208,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Replace data node and descendants: #scenario'(){ given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findExtractsWithDescendants(_, [] as Set, _) >> [] - mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath', null) + mockFragmentRepository.findByAnchorAndXpathIn(_, [] as Set) >> [] + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath'] as Set) >> [ + new FragmentEntity(1, '/test/xpath', null, '{"id":"testId"}', anchorEntity, [] as Set) ] when: 'replace data node tree' objectUnderTest.updateDataNodesAndDescendants('dataspaceName', 'anchorName', dataNodes) @@ -223,9 +224,9 @@ class CpsDataPersistenceServiceSpec extends Specification { def 'Replace data nodes and descendants'() { given: 'the fragment repository returns fragment entities related to the xpath inputs' - mockFragmentRepository.findExtractsWithDescendants(_, ['/test/xpath1', '/test/xpath2'] as Set, _) >> [ - mockFragmentExtract(1, null, 123, '/test/xpath1', null), - mockFragmentExtract(2, null, 123, '/test/xpath2', null) + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/test/xpath1', '/test/xpath2'] as Set) >> [ + new FragmentEntity(1, '/test/xpath1', null, null, anchorEntity, [] as Set), + new FragmentEntity(2, '/test/xpath2', null, null, anchorEntity, [] as Set) ] and: 'some data nodes with descendants' def dataNode1 = new DataNode(xpath: '/test/xpath1', leaves: ['id': 'testId1'], childDataNodes: [new DataNode(xpath: '/test/xpath1/child', leaves: ['id': 'childTestId1'])]) @@ -253,38 +254,27 @@ class CpsDataPersistenceServiceSpec extends Specification { def createDataNodesAndMockRepositoryMethodSupportingThem(Map<String, String> xpathToScenarioMap) { def dataNodes = [] - def fragmentExtracts = [] + def fragmentEntities = [] def fragmentId = 1 xpathToScenarioMap.each { def xpath = it.key def scenario = it.value def dataNode = new DataNodeBuilder().withXpath(xpath).build() dataNodes.add(dataNode) - def fragmentExtract = mockFragmentExtract(fragmentId, null, 123, xpath, null) - fragmentExtracts.add(fragmentExtract) def fragmentEntity = new FragmentEntity(id: fragmentId, anchor: anchorEntity, xpath: xpath, childFragments: []) + fragmentEntities.add(fragmentEntity) if ('EXCEPTION' == scenario) { mockFragmentRepository.save(fragmentEntity) >> { throw new StaleStateException("concurrent updates") } } fragmentId++ } - mockFragmentRepository.findExtractsWithDescendants(_, xpathToScenarioMap.keySet(), _) >> fragmentExtracts + mockFragmentRepository.findByAnchorAndXpathIn(_, xpathToScenarioMap.keySet()) >> fragmentEntities return dataNodes } def mockFragmentWithJson(json) { - def fragmentExtract = mockFragmentExtract(456, null, 123, '/parent-01', json) - mockFragmentRepository.findExtractsWithDescendants(123, ['/parent-01'] as Set, _) >> [fragmentExtract] - } - - def mockFragmentExtract(id, parentId, anchorId, xpath, attributes) { - def fragmentExtract = Mock(FragmentExtract) - fragmentExtract.getId() >> id - fragmentExtract.getParentId() >> parentId - fragmentExtract.getAnchorId() >> anchorId - fragmentExtract.getXpath() >> xpath - fragmentExtract.getAttributes() >> attributes - return fragmentExtract + def fragmentEntity = new FragmentEntity(456, '/parent-01', null, json, anchorEntity, [] as Set) + mockFragmentRepository.findByAnchorAndXpathIn(_, ['/parent-01'] as Set) >> [fragmentEntity] } } |