From 745386612b36f9aa33ed778887b0b5879f6598dd Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Thu, 5 Dec 2024 00:06:24 +0000 Subject: [k6] Use constant arrival rate Currently, k6 KPI tests send requests using 'constant-vus' executor. This continuously sends requests on different threads - as soon as one request completes, another is sent. This leads to weird results where improving performance of one operation ends up degrading other test cases, due to more requests being sent per second. This change makes KPI tests send requests at a constant rate, making the mixed-load tests less prone to such anomalies. - Send requests at constant rates for test cases 3, 4, 5, 6. - Throughput tests 1, 2 & 7 still send at the fastest rate. - Adjust expected performance timings. Issue-ID: CPS-2519 Signed-off-by: danielhanrahan Change-Id: I641ead7d2d9d3b29116e1cf0d4bf8cd345b01a98 --- k6-tests/ncmp/common/utils.js | 30 ++++---- k6-tests/ncmp/config/kpi.json | 156 +++++++++++++++++++++++++++--------------- 2 files changed, 116 insertions(+), 70 deletions(-) (limited to 'k6-tests/ncmp') diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index b937b4beb3..589a2482e6 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -89,21 +89,21 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) { makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig), makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 50, testResults, scenarioConfig), makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 100, testResults, scenarioConfig), - makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 500, testResults, scenarioConfig), - makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 500, testResults, scenarioConfig), - makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 1200, testResults, scenarioConfig), - makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 1200, testResults, scenarioConfig), - makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 4200, testResults, scenarioConfig), - makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 8000, testResults, scenarioConfig), - makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 10000, testResults, scenarioConfig), - makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 10000, testResults, scenarioConfig), - makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 10000, testResults, scenarioConfig), - makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 13000, testResults, scenarioConfig), - makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 30, testResults, scenarioConfig), - makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 60, testResults, scenarioConfig), - makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 30, testResults, scenarioConfig), - makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 60, testResults, scenarioConfig), - makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 1500, testResults, scenarioConfig), + makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 300, testResults, scenarioConfig), + makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 300, testResults, scenarioConfig), + makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 750, testResults, scenarioConfig), + makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 750, testResults, scenarioConfig), + makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 3000, testResults, scenarioConfig), + makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 3000, testResults, scenarioConfig), + makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 4000, testResults, scenarioConfig), + makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 4500, testResults, scenarioConfig), + makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 4500, testResults, scenarioConfig), + makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 7000, testResults, scenarioConfig), + makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 20, testResults, scenarioConfig), + makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 40, testResults, scenarioConfig), + makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 20, testResults, scenarioConfig), + makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 40, testResults, scenarioConfig), + makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 300, testResults, scenarioConfig), ]; return summaryCsvLines.join('\n') + '\n'; } diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json index 742321f709..2318702449 100644 --- a/k6-tests/ncmp/config/kpi.json +++ b/k6-tests/ncmp/config/kpi.json @@ -6,89 +6,135 @@ }, "scenarios": { "passthrough_read_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughReadScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "0ms" }, "passthrough_read_alt_id_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughReadAltIdScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "200ms" }, + "passthrough_write_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughWriteScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "400ms" }, "passthrough_write_alt_id_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughWriteAltIdScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "600ms" }, + "cm_handle_id_search_nofilter_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchNoFilterScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_nofilter_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchNoFilterScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "0ms" }, "cm_handle_id_search_module_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchModuleScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_module_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchModuleScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "400ms" }, "cm_handle_id_search_property_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchPropertyScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_property_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchPropertyScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "800ms" }, "cm_handle_id_search_cpspath_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchCpsPathScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_cpspath_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchCpsPathScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "1200ms" }, "cm_handle_id_search_trustlevel_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchTrustLevelScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "1600ms" + }, + + "cm_handle_search_nofilter_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchNoFilterScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "0s" + }, + "cm_handle_search_module_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchModuleScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "3s" + }, + "cm_handle_search_property_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchPropertyScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "6s" + }, + "cm_handle_search_cpspath_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchCpsPathScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "9s" }, "cm_handle_search_trustlevel_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleSearchTrustLevelScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "12s" }, + "legacy_batch_produce_scenario": { "executor": "shared-iterations", "exec": "legacyBatchProduceScenario", @@ -122,4 +168,4 @@ "cm_search_trustlevel_duration": ["avg <= 15000"], "legacy_batch_read_cmhandles_per_second": ["avg >= 150"] } -} \ No newline at end of file +} -- cgit