diff options
author | danielhanrahan <daniel.hanrahan@est.tech> | 2023-11-13 18:17:21 +0000 |
---|---|---|
committer | danielhanrahan <daniel.hanrahan@est.tech> | 2023-11-13 20:12:50 +0000 |
commit | 09086b5a7ce9a7467b42c6a86374f22318aaa4cd (patch) | |
tree | 6397f176821a43758d669c19cc9421858afcdfea | |
parent | ed9d179947325bbb9b81378b469341d023b46cd8 (diff) |
Adjust performance test timings
New limits for time and memory are based on analyzing 40 previous
performance job runs. Test limits are chosen so that individual
test cases would pass 95% of the the time.
Issue-ID: CPS-1884
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: Iadca27eb1f9482016759070e5afdc70c53f9d043
11 files changed, 64 insertions, 71 deletions
diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy index 8cdbdc5955..d8012ec6d4 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/CpsPerfTestBase.groovy @@ -67,7 +67,7 @@ class CpsPerfTestBase extends PerfTestBase { addAnchorsWithData(OPENROADM_ANCHORS, CPS_PERFORMANCE_TEST_DATASPACE, LARGE_SCHEMA_SET, 'openroadm', data) resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 200, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Creating openroadm anchors with large data tree', 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) } def generateOpenRoadData(numberOfNodes) { @@ -87,7 +87,7 @@ class CpsPerfTestBase extends PerfTestBase { then: 'memory used is within #peakMemoryUsage' assert resourceMeter.getTotalMemoryUsageInMB() <= 30 and: 'all data is read within expected time' - recordAndAssertResourceUsage("Warming database", 200, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Warming database", 200, durationInSeconds, 600, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy index b0a105c7fe..a96d7f64b6 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/base/PerfTestBase.groovy @@ -55,7 +55,7 @@ abstract class PerfTestBase extends CpsIntegrationSpecBase { abstract def createInitialData() def recordAndAssertResourceUsage(String shortTitle, double thresholdInSec, double recordedTimeInSec, memoryLimit, memoryUsageInMB) { - def pass = recordedTimeInSec <= thresholdInSec + def pass = recordedTimeInSec <= thresholdInSec && memoryUsageInMB <= memoryLimit if (shortTitle.length() > 40) { shortTitle = shortTitle.substring(0, 40) } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy index 6a3d4434b3..ce66cb08a2 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/CpsDataServiceLimitsPerfTest.groovy @@ -47,7 +47,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation completes within 25 seconds' - recordAndAssertResourceUsage("Creating 33,000 books", 25, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Creating 33,000 books", 25, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) } def 'Get data nodes from multiple xpaths 32K (2^15) limit exceeded.'() { @@ -88,7 +88,7 @@ class CpsDataServiceLimitsPerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'test data is deleted in 1 second' - recordAndAssertResourceUsage("Deleting test data", 1, durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Deleting test data", 1, durationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def countDataNodes() { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy index 0dcd995a77..818c300a56 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/DeletePerfTest.groovy @@ -40,7 +40,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def setupDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'setup duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete test setup', 200, setupDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete test setup', 200, setupDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 container nodes'() { @@ -56,7 +56,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 containers', 2, deleteDurationInSeconds, 30, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 containers', 2.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 container nodes'() { @@ -70,7 +70,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 containers', 0.5, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 containers', 0.6, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 list elements'() { @@ -86,7 +86,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 lists elements', 2, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 lists elements', 2.5, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 list elements'() { @@ -100,7 +100,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.5, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 lists elements', 0.6, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 100 whole lists'() { @@ -116,7 +116,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete 100 whole lists', 5, deleteDurationInSeconds, 30, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete 100 whole lists', 6, deleteDurationInSeconds, 20, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 whole lists'() { @@ -130,7 +130,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 whole lists', 4, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 whole lists', 5, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete 1 large data node'() { @@ -140,7 +140,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete one large node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete root node with many descendants'() { @@ -150,7 +150,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete root node', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Delete data nodes for an anchor'() { @@ -160,7 +160,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete data nodes for anchor', 2, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch delete 100 non-existing nodes'() { @@ -174,7 +174,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch delete 100 non-existing', 7, deleteDurationInSeconds, 5, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch delete 100 non-existing', 7, deleteDurationInSeconds, 3, resourceMeter.getTotalMemoryUsageInMB()) } def 'Clean up test data'() { @@ -186,7 +186,7 @@ class DeletePerfTest extends CpsPerfTestBase { resourceMeter.stop() def deleteDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'delete duration is within expected time and memory used is within limit' - recordAndAssertResourceUsage('Delete test cleanup', 10, deleteDurationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Delete test cleanup', 10, deleteDurationInSeconds, 1, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy index 95cf260f2a..8a228a353e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/GetPerfTest.groovy @@ -44,9 +44,9 @@ class GetPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Read datatrees with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.01 | 5 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.05 | 10 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.02 | 1 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.06 | 5 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2.5 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Read data trees for multiple xpaths'() { @@ -60,7 +60,7 @@ class GetPerfTest extends CpsPerfTestBase { then: 'requested nodes and their descendants are returned' assert countDataNodesInTree(result) == OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE and: 'all data is read within expected time and memory used is within limit' - recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 3 , durationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Read datatrees for multiple xpaths", 4 , durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) } def 'Read for multiple xpaths to non-existing datanodes'() { @@ -74,7 +74,7 @@ class GetPerfTest extends CpsPerfTestBase { then: 'no data is returned' assert result.isEmpty() and: 'the operation completes within within expected time' - recordAndAssertResourceUsage("Read non-existing xpaths", 0.01, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Read non-existing xpaths", 0.02, durationInSeconds, 2, resourceMeter.getTotalMemoryUsageInMB()) } def 'Read complete data trees using #scenario.'() { @@ -88,9 +88,9 @@ class GetPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Read datatrees using ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following xpaths are used' scenario | xpath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'openroadm root' | '/' || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm top element' | '/openroadm-devices' || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 3 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm root' | '/' || 2 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm top element' | '/openroadm-devices' || 2 | 250 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'openroadm whole list' | '/openroadm-devices/openroadm-device' || 3 | 250 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy index 5cf4955633..0ae018d462 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/QueryPerfTest.groovy @@ -45,10 +45,11 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query 1 anchor ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpsPath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 2 | 300 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 3 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 2 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 2 | 300 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'top element' | '/openroadm-devices' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 2.5 | 400 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1 + 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.1 | 1 | 0 } def 'Query complete data trees across all anchors with #scenario.'() { @@ -63,11 +64,10 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query across anchors ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | cpspath || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'top element' | '/openroadm-devices' || 6 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 6 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) - 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 6 | 800 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 6 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) - 'non-existing data' | '/path/to/non-existing/node[@id="1"]' || 0.1 | 3 | 0 + 'top element' | '/openroadm-devices' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition' | '//openroadm-device[@ne-state="inservice"]' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE) + 'ancestors' | '//openroadm-device/ancestor::openroadm-devices' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) + 'leaf condition + ancestors' | '//openroadm-device[@status="success"]/ancestor::openroadm-devices' || 7 | 600 | OPENROADM_ANCHORS * (OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 1) } def 'Query with leaf condition and #scenario.'() { @@ -82,9 +82,9 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.1 | 30 | OPENROADM_DEVICES_PER_ANCHOR - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.15 | 30 | OPENROADM_DEVICES_PER_ANCHOR * 2 - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.1 | 6 | OPENROADM_DEVICES_PER_ANCHOR + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.2 | 12 | OPENROADM_DEVICES_PER_ANCHOR * 2 + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2.5 | 200 | OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } def 'Query ancestors with #scenario.'() { @@ -99,9 +99,9 @@ class QueryPerfTest extends CpsPerfTestBase { recordAndAssertResourceUsage("Query ancestors with ${scenario}", durationLimit, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) where: 'the following parameters are used' scenario | fetchDescendantsOption || durationLimit | memoryLimit | expectedNumberOfDataNodes - 'no descendants' | OMIT_DESCENDANTS || 0.1 | 20 | 1 - 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.1 | 20 | 1 + OPENROADM_DEVICES_PER_ANCHOR - 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2 | 200 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE + 'no descendants' | OMIT_DESCENDANTS || 0.1 | 3 | 1 + 'direct descendants' | DIRECT_CHILDREN_ONLY || 0.2 | 8 | 1 + OPENROADM_DEVICES_PER_ANCHOR + 'all descendants' | INCLUDE_ALL_DESCENDANTS || 2.5 | 400 | 1 + OPENROADM_DEVICES_PER_ANCHOR * OPENROADM_DATANODES_PER_DEVICE } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy index 151492dc96..8118010b4e 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/UpdatePerfTest.groovy @@ -41,7 +41,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Update 1 data node', 0.6, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Update 1 data node', 0.6, updateDurationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch update 100 data nodes with descendants'() { @@ -57,7 +57,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Update 100 data nodes', 30, updateDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Update 100 data nodes', 40, updateDurationInSeconds, 800, resourceMeter.getTotalMemoryUsageInMB()) } def 'Update leaves for 1 data node (twice)'() { @@ -71,7 +71,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Update leaves for 1 data node', 0.5, updateDurationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Update leaves for 1 data node', 0.7, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) } def 'Batch update leaves for 100 data nodes (twice)'() { @@ -85,7 +85,7 @@ class UpdatePerfTest extends CpsPerfTestBase { resourceMeter.stop() def updateDurationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'update completes within expected time and memory used is within limit' - recordAndAssertResourceUsage('Batch update leaves for 100 data nodes', 1, updateDurationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage('Batch update leaves for 100 data nodes', 1, updateDurationInSeconds, 200, resourceMeter.getTotalMemoryUsageInMB()) } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy index 177cd9fc53..0c7107a56d 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/cps/WritePerfTest.groovy @@ -36,19 +36,16 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation takes less than #expectedDuration and memory used is within limit' - recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDurationInSeconds, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Writing ${totalNodes} devices", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) cleanup: cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') where: - totalNodes || expectedDurationInSeconds - 50 || 3 - 100 || 5 - 200 || 10 - 400 || 20 -// 800 || 40 -// 1600 || 80 -// 3200 || 160 + totalNodes || expectedDuration | memoryLimit + 50 || 4 | 100 + 100 || 7 | 200 + 200 || 14 | 400 + 400 || 28 | 500 } def 'Writing bookstore data has exponential time.'() { @@ -64,20 +61,16 @@ class WritePerfTest extends CpsPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'the operation takes less than #expectedDuration and memory used is within limit' - recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Writing ${totalBooks} books", expectedDuration, durationInSeconds, memoryLimit, resourceMeter.getTotalMemoryUsageInMB()) cleanup: cpsDataService.deleteDataNodes(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor', OffsetDateTime.now()) cpsAdminService.deleteAnchor(CPS_PERFORMANCE_TEST_DATASPACE, 'writeAnchor') where: - totalBooks || expectedDuration - 400 || 0.2 - 800 || 0.5 - 1600 || 1 - 3200 || 3 - 6400 || 10 -// 12800 || 30 -// 25600 || 120 -// 51200 || 600 + totalBooks || expectedDuration | memoryLimit + 800 || 1 | 50 + 1600 || 2 | 100 + 3200 || 6 | 150 + 6400 || 18 | 200 } } diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy index 896217a2a8..579394be85 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmDataSubscriptionsPerfTest.groovy @@ -52,7 +52,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { matches.size() == numberOfFiltersPerCmHandle * numberOfCmHandlesPerCmDataSubscription and: 'query all subscribers within 1 second' def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - recordAndAssertResourceUsage("Query all subscribers", 1, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Query all subscribers", 1.2, durationInSeconds, 300, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case subscription update (200x10 matching entries).'() { @@ -94,8 +94,8 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { then: 'a subscriber has been added to each filter entry' def resultAfter = objectUnderTest.queryDataNodes(NCMP_PERFORMANCE_TEST_DATASPACE, CM_DATA_SUBSCRIPTIONS_ANCHOR, cpsPath, INCLUDE_ALL_DESCENDANTS) assert resultAfter.collect {it.leaves.subscribers.size()}.sum() == totalNumberOfEntries * (1 + numberOfCmDataSubscribers) - and: 'update matching subscription within 8 seconds' - recordAndAssertResourceUsage("Update matching subscription", 8, durationInSeconds, 400, resourceMeter.getTotalMemoryUsageInMB()) + and: 'update matching subscription within 15 seconds' + recordAndAssertResourceUsage("Update matching subscription", 15, durationInSeconds, 1000, resourceMeter.getTotalMemoryUsageInMB()) } def 'Worst case new subscription (200x10 new entries).'() { @@ -109,7 +109,7 @@ class CmDataSubscriptionsPerfTest extends NcmpPerfTestBase { resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() then: 'insert new subscription with 1 second' - recordAndAssertResourceUsage("Insert new subscription", 1, durationInSeconds, 400,resourceMeter.getTotalMemoryUsageInMB()) + recordAndAssertResourceUsage("Insert new subscription", 2, durationInSeconds, 100, resourceMeter.getTotalMemoryUsageInMB()) } def querySubscriptionsByIteration(Collection<DataNode> allSubscriptionsAsDataNodes, targetSubscriptionSequenceNumber) { diff --git a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy index becd7e39c4..a5a6acb7a1 100644 --- a/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy +++ b/integration-test/src/test/groovy/org/onap/cps/integration/performance/ncmp/CmHandleQueryPerfTest.groovy @@ -45,8 +45,8 @@ class CmHandleQueryPerfTest extends NcmpPerfTestBase { def result = cpsDataService.getDataNodesForMultipleXpaths(NCMP_PERFORMANCE_TEST_DATASPACE, REGISTRY_ANCHOR, xpaths, INCLUDE_ALL_DESCENDANTS) resourceMeter.stop() def durationInSeconds = resourceMeter.getTotalTimeInSeconds() - then: 'the required operations are performed within 1200 ms' - recordAndAssertResourceUsage("CpsPath Registry attributes Query", 0.25, durationInSeconds, 150, resourceMeter.getTotalMemoryUsageInMB()) + then: 'the required operations are performed within required time' + recordAndAssertResourceUsage("CpsPath Registry attributes Query", 0.4, durationInSeconds, 50, resourceMeter.getTotalMemoryUsageInMB()) and: 'all but 1 (other node) are returned' result.size() == 999 and: 'the tree contains all the expected descendants too' diff --git a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java index 1e420013d4..c7d96c4c2b 100644 --- a/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java +++ b/integration-test/src/test/java/org/onap/cps/integration/ResourceMeter.java @@ -48,8 +48,8 @@ public class ResourceMeter { } /** - * Get the total time in milliseconds. - * @return total time in milliseconds + * Get the total time in seconds. + * @return total time in seconds */ public double getTotalTimeInSeconds() { return stopWatch.getTotalTimeSeconds(); |