diff options
author | Toine Siebelink <toine.siebelink@est.tech> | 2023-10-17 13:17:08 +0000 |
---|---|---|
committer | Gerrit Code Review <gerrit@onap.org> | 2023-10-17 13:17:08 +0000 |
commit | c29a5fe32db7efb0fdefa3a3d020110aab9731b4 (patch) | |
tree | 0e01843a2cb336d8a0045fe77581e7e94fbbc0b4 /integration-test/src | |
parent | fc7a5d30953cfedd7e2ea2f969adab03716de6df (diff) | |
parent | 500134c9c745eeda707e5738a5a699c69bb899c6 (diff) |
Merge "Add memory usage to integration tests [UPDATED]"
Diffstat (limited to 'integration-test/src')
12 files changed, 273 insertions, 201 deletions
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy index 5fd27476c0..e7a847e13e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy @@ -20,6 +20,7 @@ package org.onap.cps.integration.performance.base +import org.onap.cps.integration.ResourceMeter import org.onap.cps.rest.utils.MultipartFileUtil import org.onap.cps.spi.FetchDescendantsOption import org.springframework.web.multipart.MultipartFile @@ -33,6 +34,8 @@ class CpsPerfTestBase extends PerfTestBase { static final def OPENROADM_DEVICES_PER_ANCHOR = 1000 static final def OPENROADM_DATANODES_PER_DEVICE = 86 + ResourceMeter resourceMeter = new ResourceMeter() + def printTitle() { println('## C P S P E R F O R M A N C E T E S T R E S U L T S ##') } @@ -62,11 +65,11 @@ class CpsPerfTestBase extends PerfTestBase { def addOpenRoadData() { def data = generateOpenRoadData(OPENROADM_DEVICES_PER_ANCHOR) - stopWatch.start() + resourceMeter.start() addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() - recordAndAssertPerformance('Creating openroadm anchors with large data tree', TimeUnit.SECONDS.toMillis(200), durationInMillis) + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() + recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', TimeUnit.SECONDS.toMillis(200), durationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } def generateOpenRoadData(numberOfNodes) { @@ -78,13 +81,15 @@ class CpsPerfTestBase extends PerfTestBase { def 'Warm the database'() { when: 'dummy get data nodes runs so that populating the DB does not get included in other test timings' - stopWatch.start() + resourceMeter.start() def result = cpsDataService.getDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm1', '/', FetchDescendantsOption.OMIT_DESCENDANTS) assert countDataNodesInTree(result) == 1 - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() - then: 'all data is read within expected time' - recordAndAssertPerformance("Warming database", TimeUnit.SECONDS.toMillis(200), durationInMillis) + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() + then: 'memory used is within #peakMemoryUsage' + assert resourceMeter.getTotalMemoryUsageInMB() <= 30 + and: 'all data is read within expected time' + recordAndAssertResourceUsage("Warming database", TimeUnit.SECONDS.toMillis(200), durationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy index f5d7c5e156..7bacf1d6e9 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/NcmpPerfTestBase.groovy @@ -21,6 +21,7 @@ package org.onap.cps.integration.performance.base import java.time.OffsetDateTime +import org.onap.cps.integration.ResourceMeter class NcmpPerfTestBase extends PerfTestBase { @@ -36,6 +37,8 @@ class NcmpPerfTestBase extends PerfTestBase { def numberOfFiltersPerCmHandle = 10 def numberOfCmHandlesPerCmDataSubscription = 200 + ResourceMeter resourceMeter = new ResourceMeter() + // SHORT versions for easier debugging // def subscriberIdPrefix = 'sub' // def xpathPrefix = 'f' diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy index b6ceb91b5a..8e5fe06196 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy @@ -28,8 +28,6 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { static def LARGE_SCHEMA_SET = 'largeSchemaSet' static def PERFORMANCE_RECORD = [] - def stopWatch = new StopWatch() - def cleanupSpec() { println('#############################################################################') printTitle() @@ -56,15 +54,16 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { abstract def createInitialData() - def recordAndAssertPerformance(String shortTitle, thresholdInMs, recordedTimeInMs) { + def recordAndAssertResourceUsage(String shortTitle, thresholdInMs, recordedTimeInMs, memoryLimit, memoryUsageInMB) { def pass = recordedTimeInMs <= thresholdInMs if (shortTitle.length() > 40) { shortTitle = shortTitle.substring(0, 40) } - def record = String.format('%2d.%-40s limit%,8d took %,8d ms ', PERFORMANCE_RECORD.size() + 1, shortTitle, thresholdInMs, recordedTimeInMs) + def record = String.format('%2d.%-40s limit%,8d took %,8d ms %,8.2f MB used ', PERFORMANCE_RECORD.size() + 1, shortTitle, thresholdInMs, recordedTimeInMs, memoryUsageInMB) record += pass ? 'PASS' : 'FAIL' PERFORMANCE_RECORD.add(record) assert recordedTimeInMs <= thresholdInMs + assert memoryUsageInMB <= memoryLimit return true } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy index 659c9f5792..1aea1235e2 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy @@ -41,15 +41,15 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { def parentNodeData = '{"bookstore": { "categories": [{ "code": 1, "name": "Test", "books" : [] }] }}' cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor', parentNodeData, OffsetDateTime.now()) when: '33,000 books are added' - stopWatch.start() + resourceMeter.start() for (int i = 1; i <= 33_000; i+=100) { def booksData = '{"books":[' + (i..<i+100).collect {'{ "title": "' + it + '" }' }.join(',') + ']}' cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor', '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now()) } - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'the operation completes within 25 seconds' - recordAndAssertPerformance("Creating 33,000 books", TimeUnit.SECONDS.toMillis(25), durationInMillis) + recordAndAssertResourceUsage("Creating 33,000 books", TimeUnit.SECONDS.toMillis(25), durationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() { @@ -84,13 +84,13 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { def 'Clean up test data.'() { when: - stopWatch.start() + resourceMeter.start() cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor', OffsetDateTime.now()) cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'limitsAnchor') - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'test data is deleted in 1 second' - recordAndAssertPerformance("Deleting test data", TimeUnit.SECONDS.toMillis(1), durationInMillis) + recordAndAssertResourceUsage("Deleting test data", TimeUnit.SECONDS.toMillis(1), durationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } def countDataNodes() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy index 0bab6159d4..91c1082a89 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy @@ -36,13 +36,13 @@ class DeletePerfTest extends CpsPerfTestBase { def 'Create test data (please note, subsequent tests depend on this running first).'() { when: 'multiple anchors with a node with a large number of descendants is created' - stopWatch.start() + resourceMeter.start() def data = generateOpenRoadData(300) addAnchorsWithData(10, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'delete', data) - stopWatch.stop() - def setupDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'setup duration is within expected time' - recordAndAssertPerformance('Delete test setup', TimeUnit.SECONDS.toMillis(200), setupDurationInMillis) + resourceMeter.stop() + def setupDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'setup duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete test setup', TimeUnit.SECONDS.toMillis(200), setupDurationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 container nodes'() { @@ -51,14 +51,14 @@ class DeletePerfTest extends CpsPerfTestBase { "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-" + it + "']/org-openroadm-device" } when: 'child nodes are deleted' - stopWatch.start() + resourceMeter.start() xpathsToDelete.each { objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete1', it, OffsetDateTime.now()) } - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Delete 100 containers', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete 100 containers', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis, 30, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 container nodes'() { @@ -67,12 +67,12 @@ class DeletePerfTest extends CpsPerfTestBase { "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-" + it + "']/org-openroadm-device" } when: 'child nodes are deleted' - stopWatch.start() + resourceMeter.start() objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete2', xpathsToDelete, OffsetDateTime.now()) - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Batch delete 100 containers', 500, deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Batch delete 100 containers', 500, deleteDurationInMillis, 5, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 list elements'() { @@ -81,14 +81,14 @@ class DeletePerfTest extends CpsPerfTestBase { "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-" + it + "']" } when: 'list elements are deleted' - stopWatch.start() + resourceMeter.start() xpathsToDelete.each { objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete3', it, OffsetDateTime.now()) } - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Delete 100 lists elements', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete 100 lists elements', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 list elements'() { @@ -97,12 +97,12 @@ class DeletePerfTest extends CpsPerfTestBase { "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-" + it + "']" } when: 'list elements are deleted' - stopWatch.start() + resourceMeter.start() objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete4', xpathsToDelete, OffsetDateTime.now()) - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Batch delete 100 lists elements', 500, deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Batch delete 100 lists elements', 500, deleteDurationInMillis, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 whole lists'() { @@ -111,14 +111,14 @@ class DeletePerfTest extends CpsPerfTestBase { "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-" + it + "']/org-openroadm-device/degree" } when: 'lists are deleted' - stopWatch.start() + resourceMeter.start() xpathsToDelete.each { objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete5', it, OffsetDateTime.now()) } - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Delete 100 whole lists', TimeUnit.SECONDS.toMillis(5), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete 100 whole lists', TimeUnit.SECONDS.toMillis(5), deleteDurationInMillis, 30, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 whole lists'() { @@ -127,68 +127,68 @@ class DeletePerfTest extends CpsPerfTestBase { "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-" + it + "']/org-openroadm-device/degree" } when: 'lists are deleted' - stopWatch.start() + resourceMeter.start() objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete6', xpathsToDelete, OffsetDateTime.now()) - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Batch delete 100 whole lists', TimeUnit.SECONDS.toMillis(4), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Batch delete 100 whole lists', TimeUnit.SECONDS.toMillis(4), deleteDurationInMillis, 5, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 1 large data node'() { when: 'parent node is deleted' - stopWatch.start() + resourceMeter.start() objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete7', '/openroadm-devices', OffsetDateTime.now()) - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Delete one large node', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete one large node', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete root node with many descendants'() { when: 'root node is deleted' - stopWatch.start() + resourceMeter.start() objectUnderTest.deleteDataNode(CPS_PERFORMANCE_TEST_DATASPACE, 'delete8', '/', OffsetDateTime.now()) - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Delete root node', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete root node', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete data nodes for an anchor'() { when: 'data nodes are deleted' - stopWatch.start() + resourceMeter.start() objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete9', OffsetDateTime.now()) - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Delete data nodes for anchor', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete data nodes for anchor', TimeUnit.SECONDS.toMillis(2), deleteDurationInMillis, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 non-existing nodes'() { given: 'a list of xpaths to delete' def xpathsToDelete = (1..100).collect { "/path/to/non-existing/node[@id='" + it + "']" } when: 'child nodes are deleted' - stopWatch.start() + resourceMeter.start() try { objectUnderTest.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'delete10', xpathsToDelete, OffsetDateTime.now()) } catch (DataNodeNotFoundException ignored) {} - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Batch delete 100 non-existing', TimeUnit.SECONDS.toMillis(7), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Batch delete 100 non-existing', TimeUnit.SECONDS.toMillis(7), deleteDurationInMillis, 5, resourceMeter.getTotalMemoryUsageInMB()) } def 'Clean up test data'() { given: 'a list of anchors to delete' def anchorNames = (1..10).collect {'delete' + it} when: 'data nodes are deleted' - stopWatch.start() + resourceMeter.start() cpsAdminService.deleteAnchors(CPS_PERFORMANCE_TEST_DATASPACE, anchorNames) - stopWatch.stop() - def deleteDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'delete duration is within expected time' - recordAndAssertPerformance('Delete test cleanup', TimeUnit.SECONDS.toMillis(10), deleteDurationInMillis) + resourceMeter.stop() + def deleteDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'delete duration is within expected time and memory used is within limit' + recordAndAssertResourceUsage('Delete test cleanup', TimeUnit.SECONDS.toMillis(10), deleteDurationInMillis, 2, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy index 048b3b4202..8b76ef5f6f 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy @@ -22,9 +22,7 @@ package org.onap.cps.integration.performance.cps import org.onap.cps.api.CpsDataService import org.onap.cps.integration.performance.base.CpsPerfTestBase - import java.util.concurrent.TimeUnit - import static org.onap.cps.spi.FetchDescendantsOption.DIRECT_CHILDREN_ONLY import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS @@ -37,62 +35,62 @@ class GetPerfTest extends CpsPerfTestBase { def 'Read top-level node with #scenario.'() { when: 'get data nodes from 1 anchor' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.getDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm1', '/openroadm-devices', fetchDescendantsOption) - stopWatch.stop() + resourceMeter.stop() assert countDataNodesInTree(result) == expectedNumberOfDataNodes - def durationInMillis = stopWatch.getTotalTimeMillis() - then: 'all data is read within #durationLimit ms' - recordAndAssertPerformance("Read datatrees with ${scenario}", durationLimit, durationInMillis) + def durationInMillis = resourceMeter.getTotalTimeMillis() + then: 'all data is read within #durationLimit ms and memory used is within limit' + recordAndAssertResourceUsage("Read datatrees with ${scenario}", durationLimit, durationInMillis, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | fetchDescendantsOption || durationLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 10 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 50 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || TimeUnit.SECONDS.toMillis(2) | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes + 'no descendants' | OMIT_DESCENDANTS || 10 | 5 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 50 | 10 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || TimeUnit.SECONDS.toMillis(2) | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Read data trees for multiple xpaths'() { given: 'a collection of xpaths to get' def xpaths = (1..OPENROADM_DEVICES_PER_ANCHOR).collect { "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-" + it + "']" } when: 'get data nodes from 1 anchor' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.getDataNodesForMultipleXpaths(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', xpaths, INCLUDE_ALL_DESCENDANTS) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'requested nodes and their descendants are returned' assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - and: 'all data is read within expected time' - recordAndAssertPerformance("Read datatrees for multiple xpaths", TimeUnit.SECONDS.toMillis(3) , durationInMillis) + and: 'all data is read within expected time and memory used is within limit' + recordAndAssertResourceUsage("Read datatrees for multiple xpaths", TimeUnit.SECONDS.toMillis(3) , durationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } def 'Read for multiple xpaths to non-existing datanodes'() { given: 'a collection of xpaths to get' def xpaths = (1..50).collect { "/path/to/non-existing/node[@id='" + it + "']" } when: 'get data nodes from 1 anchor' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.getDataNodesForMultipleXpaths(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', xpaths, INCLUDE_ALL_DESCENDANTS) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'no data is returned' assert result.isEmpty() and: 'the operation completes within within expected time' - recordAndAssertPerformance("Read non-existing xpaths", 10, durationInMillis) + recordAndAssertResourceUsage("Read non-existing xpaths", 10, durationInMillis, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Read complete data trees using #scenario.'() { when: 'get data nodes from 1 anchor' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.getDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm3', xpath, INCLUDE_ALL_DESCENDANTS) assert countDataNodesInTree(result) == expectedNumberOfDataNodes - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() - then: 'all data is read within expected time' - recordAndAssertPerformance("Read datatrees using ${scenario}", durationLimit, durationInMillis) + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() + then: 'all data is read within expected time and memory used is within limit' + recordAndAssertResourceUsage("Read datatrees using ${scenario}", durationLimit, durationInMillis, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following xpaths are used' - scenario | xpath || durationLimit | expectedNumberOfDataNodes - 'openroadm root' | '/' || TimeUnit.SECONDS.toMillis(2) | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm top element' | '/openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm whole list' | '/openroadm-devices/openroadm-device' || TimeUnit.SECONDS.toMillis(3) | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | xpath || durationLimit | memoryLimit | expectedNumberOfDataNodes + 'openroadm root' | '/' || TimeUnit.SECONDS.toMillis(2) | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm top element' | '/openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm whole list' | '/openroadm-devices/openroadm-device' || TimeUnit.SECONDS.toMillis(3) | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index 01369181e8..595d0388df 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -23,9 +23,7 @@ package org.onap.cps.integration.performance.cps import org.onap.cps.api.CpsQueryService import org.onap.cps.integration.performance.base.CpsPerfTestBase import org.onap.cps.spi.PaginationOption - import java.util.concurrent.TimeUnit - import static org.onap.cps.spi.FetchDescendantsOption.DIRECT_CHILDREN_ONLY import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS @@ -33,79 +31,77 @@ import static org.onap.cps.spi.FetchDescendantsOption.OMIT_DESCENDANTS class QueryPerfTest extends CpsPerfTestBase { CpsQueryService objectUnderTest - def setup() { objectUnderTest = cpsQueryService } def 'Query complete data trees with #scenario.'() { when: 'query data nodes (using a fresh anchor with identical data for each test)' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.queryDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm1', cpsPath, INCLUDE_ALL_DESCENDANTS) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit ms' - recordAndAssertPerformance("Query 1 anchor ${scenario}", durationLimit, durationInMillis) + and: 'all data is read within #durationLimit ms and memory used is within limit' + recordAndAssertResourceUsage("Query 1 anchor ${scenario}", durationLimit, durationInMillis, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | cpsPath || durationLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || TimeUnit.SECONDS.toMillis(3) | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 100 | 0 + scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfDataNodes + 'top element' | '/openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | 300 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || TimeUnit.SECONDS.toMillis(3) | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(2) | 300 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 } def 'Query complete data trees across all anchors with #scenario.'() { when: 'query data nodes across all anchors' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.queryDataNodesAcrossAnchors(CPS_PERFORMANCE_TEST_DATASPACE, cpspath, INCLUDE_ALL_DESCENDANTS, PaginationOption.NO_PAGINATION) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit ms' - recordAndAssertPerformance("Query across anchors ${scenario}", durationLimit, durationInMillis) + and: 'all data is read within #durationLimit ms and memory used is within limit' + recordAndAssertResourceUsage("Query across anchors ${scenario}", durationLimit, durationInMillis, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | cpspath || durationLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || TimeUnit.SECONDS.toMillis(6) | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || TimeUnit.SECONDS.toMillis(6) | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(6) | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(6) | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 100 | 0 + scenario | cpspath || durationLimit | memoryLimit | expectedNumberOfDataNodes + 'top element' | '/openroadm-devices' || TimeUnit.SECONDS.toMillis(6) | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || TimeUnit.SECONDS.toMillis(6) | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(6) | 800 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || TimeUnit.SECONDS.toMillis(6) | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 100 | 3 | 0 } def 'Query with leaf condition and #scenario.'() { when: 'query data nodes (using a fresh anchor with identical data for each test)' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.queryDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', '//openroadm-device[@status="success"]', fetchDescendantsOption) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit ms' - recordAndAssertPerformance("Query with ${scenario}", durationLimit, durationInMillis) + and: 'all data is read within #durationLimit ms and memory used is within limit' + recordAndAssertResourceUsage("Query with ${scenario}", durationLimit, durationInMillis, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | fetchDescendantsOption || durationLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 100 | OPENROADM_DEVICES_PER_ANCHOR - 'direct descendants' | DIRECT_CHILDREN_ONLY || 150 | OPENROADM_DEVICES_PER_ANCHOR * 2 - 'all descendants' | INCLUDE_ALL_DESCENDANTS || TimeUnit.SECONDS.toMillis(2) | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes + 'no descendants' | OMIT_DESCENDANTS || 100 | 30 | OPENROADM_DEVICES_PER_ANCHOR + 'direct descendants' | DIRECT_CHILDREN_ONLY || 150 | 30 | OPENROADM_DEVICES_PER_ANCHOR * 2 + 'all descendants' | INCLUDE_ALL_DESCENDANTS || TimeUnit.SECONDS.toMillis(2) | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Query ancestors with #scenario.'() { when: 'query data nodes (using a fresh anchor with identical data for each test)' - stopWatch.start() + resourceMeter.start() def result = objectUnderTest.queryDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm3', '//openroadm-device[@ne-state="inservice"]/ancestor::openroadm-devices', fetchDescendantsOption) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'the expected number of nodes is returned' assert countDataNodesInTree(result) == expectedNumberOfDataNodes - and: 'all data is read within #durationLimit ms' - recordAndAssertPerformance("Query ancestors with ${scenario}", durationLimit, durationInMillis) + and: 'all data is read within #durationLimit ms and memory used is within limit' + recordAndAssertResourceUsage("Query ancestors with ${scenario}", durationLimit, durationInMillis, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' - scenario | fetchDescendantsOption || durationLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 100 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 100 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || TimeUnit.SECONDS.toMillis(2) | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes + 'no descendants' | OMIT_DESCENDANTS || 100 | 20 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 100 | 20 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || TimeUnit.SECONDS.toMillis(2) | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index 5bb8192e57..3e26ac154e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -23,7 +23,6 @@ package org.onap.cps.integration.performance.cps import java.time.OffsetDateTime import org.onap.cps.api.CpsDataService import org.onap.cps.integration.performance.base.CpsPerfTestBase - import java.util.concurrent.TimeUnit class UpdatePerfTest extends CpsPerfTestBase { @@ -38,12 +37,12 @@ class UpdatePerfTest extends CpsPerfTestBase { def parentNodeXpath = "/openroadm-devices/openroadm-device[@device-id='C201-7-1A-10']" def jsonData = readResourceDataFile('openroadm/innerNode.json').replace('NODE_ID_HERE', '10') when: 'the fragment entities are updated by the data nodes' - stopWatch.start() + resourceMeter.start() objectUnderTest.updateDataNodeAndDescendants(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm1', parentNodeXpath, jsonData, now) - stopWatch.stop() - def updateDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'update completes within expected time' - recordAndAssertPerformance('Update 1 data node', 600, updateDurationInMillis) + resourceMeter.stop() + def updateDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'update completes within expected time and memory used is within limit' + recordAndAssertResourceUsage('Update 1 data node', 600, updateDurationInMillis, 200, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch update 100 data nodes with descendants'() { @@ -54,12 +53,12 @@ class UpdatePerfTest extends CpsPerfTestBase { innerNodeJson.replace('NODE_ID_HERE', it.toString()) ]} when: 'the fragment entities are updated by the data nodes' - stopWatch.start() + resourceMeter.start() objectUnderTest.updateDataNodesAndDescendants(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm1', nodesJsonData, now) - stopWatch.stop() - def updateDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'update completes within expected time' - recordAndAssertPerformance('Update 100 data nodes', TimeUnit.SECONDS.toMillis(30), updateDurationInMillis) + resourceMeter.stop() + def updateDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'update completes within expected time and memory used is within limit' + recordAndAssertResourceUsage('Update 100 data nodes', TimeUnit.SECONDS.toMillis(30), updateDurationInMillis, 800, resourceMeter.getTotalMemoryUsageInMB()) } def 'Update leaves for 1 data node (twice)'() { @@ -67,13 +66,13 @@ class UpdatePerfTest extends CpsPerfTestBase { def jsonDataUpdated = "{'openroadm-device':{'device-id':'C201-7-1A-10','status':'fail','ne-state':'jeopardy'}}" def jsonDataOriginal = "{'openroadm-device':{'device-id':'C201-7-1A-10','status':'success','ne-state':'inservice'}}" when: 'update is performed for leaves' - stopWatch.start() + resourceMeter.start() objectUnderTest.updateNodeLeaves(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', "/openroadm-devices", jsonDataUpdated, now) objectUnderTest.updateNodeLeaves(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', "/openroadm-devices", jsonDataOriginal, now) - stopWatch.stop() - def updateDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'update completes within expected time' - recordAndAssertPerformance('Update leaves for 1 data node', 500, updateDurationInMillis) + resourceMeter.stop() + def updateDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'update completes within expected time and memory used is within limit' + recordAndAssertResourceUsage('Update leaves for 1 data node', 500, updateDurationInMillis, 300, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch update leaves for 100 data nodes (twice)'() { @@ -81,13 +80,13 @@ class UpdatePerfTest extends CpsPerfTestBase { def jsonDataUpdated = "{'openroadm-device':[" + (1..100).collect { "{'device-id':'C201-7-1A-" + it + "','status':'fail','ne-state':'jeopardy'}" }.join(",") + "]}" def jsonDataOriginal = "{'openroadm-device':[" + (1..100).collect { "{'device-id':'C201-7-1A-" + it + "','status':'success','ne-state':'inservice'}" }.join(",") + "]}" when: 'update is performed for leaves' - stopWatch.start() + resourceMeter.start() objectUnderTest.updateNodeLeaves(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', "/openroadm-devices", jsonDataUpdated, now) objectUnderTest.updateNodeLeaves(CPS_PERFORMANCE_TEST_DATASPACE, 'openroadm2', "/openroadm-devices", jsonDataOriginal, now) - stopWatch.stop() - def updateDurationInMillis = stopWatch.getTotalTimeMillis() - then: 'update completes within expected time' - recordAndAssertPerformance('Batch update leaves for 100 data nodes', TimeUnit.SECONDS.toMillis(1), updateDurationInMillis) + resourceMeter.stop() + def updateDurationInMillis = resourceMeter.getTotalTimeMillis() + then: 'update completes within expected time and memory used is within limit' + recordAndAssertResourceUsage('Batch update leaves for 100 data nodes', TimeUnit.SECONDS.toMillis(1), updateDurationInMillis, 300, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index d03aec2d56..f9f0986133 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -33,12 +33,12 @@ class WritePerfTest extends CpsPerfTestBase { and: 'a list of device nodes to add' def jsonData = generateOpenRoadData(totalNodes) when: 'device nodes are added' - stopWatch.start() + resourceMeter.start() cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', jsonData, OffsetDateTime.now()) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() - then: 'the operation takes less than #expectedDuration' - recordAndAssertPerformance("Writing ${totalNodes} devices", TimeUnit.SECONDS.toMillis(expectedDurationInSeconds), durationInMillis) + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() + then: 'the operation takes less than #expectedDuration and memory used is within limit' + recordAndAssertResourceUsage("Writing ${totalNodes} devices", TimeUnit.SECONDS.toMillis(expectedDurationInSeconds), durationInMillis, 400, resourceMeter.getTotalMemoryUsageInMB()) cleanup: cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') @@ -61,12 +61,12 @@ class WritePerfTest extends CpsPerfTestBase { and: 'a list of books to add' def booksData = '{"books":[' + (1..totalBooks).collect {'{ "title": "' + it + '" }' }.join(',') + ']}' when: 'books are added' - stopWatch.start() + resourceMeter.start() cpsDataService.saveData(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', '/bookstore/categories[@code=1]', booksData, OffsetDateTime.now()) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() - then: 'the operation takes less than #expectedDuration' - recordAndAssertPerformance("Writing ${totalBooks} books", expectedDuration, durationInMillis) + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() + then: 'the operation takes less than #expectedDuration and memory used is within limit' + recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInMillis, 400, resourceMeter.getTotalMemoryUsageInMB()) cleanup: cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy index cf5c3f6894..b12dbaa3ce 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy @@ -41,18 +41,18 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { def 'Find many subscribers in large dataset.'() { when: 'all filters are queried' - stopWatch.start() + resourceMeter.start() def cpsPath = '//filter' def result = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) then: 'got all filter entries' result.size() == totalNumberOfEntries then: 'find a random subscriptions by iteration (worst case: whole subscription matches previous entries)' def matches = querySubscriptionsByIteration(result, -1) - stopWatch.stop() + resourceMeter.stop() matches.size() == numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription and: 'query all subscribers within 1 second' - def durationInMillis = stopWatch.getTotalTimeMillis() - recordAndAssertPerformance("Query all subscribers", 1_000, durationInMillis) + def durationInMillis = resourceMeter.getTotalTimeMillis() + recordAndAssertResourceUsage("Query all subscribers", 1_000, durationInMillis, 400, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case subscription update (200x10 matching entries).'() { @@ -64,7 +64,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { and: 'find all entries for an existing subscriptions' def matches = querySubscriptionsByIteration(result, 1) when: 'update all subscriptions found' - stopWatch.start() + resourceMeter.start() HashMap<String, List<String>> filterEntriesPerPath = [:] matches.each { dataNode, subscribersAsArray -> def updatedSubscribers = createLeafList('subscribers', 1 + numberOfCmDataSubscribers, subscriberIdPrefix) @@ -89,13 +89,13 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { cpsDataService.updateNodeLeaves(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, parentPath, json, now) } - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'a subscriber has been added to each filter entry' def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers) and: 'update matching subscription within 8 seconds' - recordAndAssertPerformance("Update matching subscription", 8_000, durationInMillis) + recordAndAssertResourceUsage("Update matching subscription", 8_000, durationInMillis, 400, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case new subscription (200x10 new entries).'() { @@ -104,12 +104,12 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { def filters = '"filters":' + createJsonArray('filter',numberOfFiltersPerCmHandle,'xpath','other_' + xpathPrefix,subscribers) def cmHandles = createJsonArray('cm-handle',numberOfCmHandlesPerCmDataSubscription,'id','other' + cmHandlePrefix, filters) when: 'Insert a new subscription' - stopWatch.start() + resourceMeter.start() cpsDataService.saveData(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, xPathForDataStore1CmHandles, cmHandles, now) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'insert new subscription with 1 second' - recordAndAssertPerformance("Insert new subscription", 1_000, durationInMillis) + recordAndAssertResourceUsage("Insert new subscription", 1_000, durationInMillis, 400,resourceMeter.getTotalMemoryUsageInMB()) } def querySubscriptionsByIteration(Collection<DataNode> allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) { @@ -118,7 +118,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { String[] subscribersAsArray = it.leaves.get('subscribers') Set<String> subscribersAsSet = new HashSet<>(Arrays.asList(subscribersAsArray)) def targetSubscriptionId = subscriberIdPrefix + '-' + ( targetSubscriptionSequenceNumber > 0 ? targetSubscriptionSequenceNumber - : 1 + random.nextInt(numberOfCmDataSubscribers) ) + : 1 + random.nextInt(numberOfCmDataSubscribers) ) if (subscribersAsSet.contains(targetSubscriptionId)) { matches.put(it, subscribersAsArray) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index 54e56d873a..02881fec1c 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -20,6 +20,7 @@ package org.onap.cps.integration.performance.ncmp +import org.onap.cps.integration.ResourceMeter import java.util.stream.Collectors import org.onap.cps.api.CpsQueryService import org.onap.cps.integration.performance.base.NcmpPerfTestBase @@ -29,22 +30,23 @@ import static org.onap.cps.spi.FetchDescendantsOption.INCLUDE_ALL_DESCENDANTS class CmHandleQueryPerfTest extends NcmpPerfTestBase { CpsQueryService objectUnderTest + ResourceMeter resourceMeter = new ResourceMeter() def setup() { objectUnderTest = cpsQueryService } def 'Query CM Handle IDs by a property name and value.'() { when: 'a cps-path query on name-value pair is performed (without getting descendants)' - stopWatch.start() + resourceMeter.start() def cpsPath = '//additional-properties[@name="neType" and @value="RadioNode"]/ancestor::cm-handles' def dataNodes = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, cpsPath, OMIT_DESCENDANTS) and: 'the ids of the result are extracted and converted to xpath' def xpaths = dataNodes.stream().map(dataNode -> "/dmi-registry/cm-handles[@id='${dataNode.leaves.id}']".toString() ).collect(Collectors.toSet()) and: 'a single get is executed to get all the parent objects and their descendants' def result = cpsDataService.getDataNodesForMultipleXpaths(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, xpaths, INCLUDE_ALL_DESCENDANTS) - stopWatch.stop() - def durationInMillis = stopWatch.getTotalTimeMillis() + resourceMeter.stop() + def durationInMillis = resourceMeter.getTotalTimeMillis() then: 'the required operations are performed within 1200 ms' - recordAndAssertPerformance("CpsPath Registry attributes Query", 250, durationInMillis) + recordAndAssertResourceUsage("CpsPath Registry attributes Query", 250, durationInMillis, 150, resourceMeter.getTotalMemoryUsageInMB()) and: 'all but 1 (other node) are returned' result.size() == 999 and: 'the tree contains all the expected descendants too' diff --git a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java new file mode 100644 index 0000000000..c6ad96e917 --- /dev/null +++ b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java @@ -0,0 +1,70 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2023 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +package org.onap.cps.integration; + +import org.springframework.util.StopWatch; + +/** + * Time and memory stop watch, exposing total running time and memory used. + */ +public class ResourceMeter { + private final StopWatch stopWatch = new StopWatch(); + private long memoryUsedBefore; + private long memoryUsedAfter; + + /** + * Start measurement. + */ + public void start() { + System.gc(); + memoryUsedBefore = getCurrentMemoryUsage(); + stopWatch.start(); + } + + /** + * Stop measurement. + */ + public void stop() { + stopWatch.stop(); + memoryUsedAfter = getCurrentMemoryUsage(); + } + + /** + * Get the total time in milliseconds. + * @return total time in milliseconds + */ + public long getTotalTimeMillis() { + return stopWatch.getTotalTimeMillis(); + } + + /** + * Get the total memory used in megabytes. + * @return total memory used in megabytes + */ + public double getTotalMemoryUsageInMB() { + return (memoryUsedAfter - memoryUsedBefore) / 1_000_000.0; + } + + private static long getCurrentMemoryUsage() { + return Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(); + } +} + |