[k6] Use constant arrival rate

Currently, k6 KPI tests send requests using 'constant-vus' executor.
This continuously sends requests on different threads - as soon as
one request completes, another is sent. This leads to weird results
where improving performance of one operation ends up degrading other
test cases, due to more requests being sent per second.

This change makes KPI tests send requests at a constant rate,
making the mixed-load tests less prone to such anomalies.

- Send requests at constant rates for test cases 3, 4, 5, 6.
- Throughput tests 1, 2 & 7 still send at the fastest rate.
- Adjust expected performance timings.

Issue-ID: CPS-2519
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Change-Id: I641ead7d2d9d3b29116e1cf0d4bf8cd345b01a98

2 files changed, 116 insertions, 70 deletions

diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index b937b4beb3..589a2482e6 100644
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -89,21 +89,21 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) {
         makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig),
         makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 50, testResults, scenarioConfig),
         makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 100, testResults, scenarioConfig),
-        makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 500, testResults, scenarioConfig),
-        makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 500, testResults, scenarioConfig),
-        makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 1200, testResults, scenarioConfig),
-        makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 1200, testResults, scenarioConfig),
-        makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 4200, testResults, scenarioConfig),
-        makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 8000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 10000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 10000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 10000, testResults, scenarioConfig),
-        makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 13000, testResults, scenarioConfig),
-        makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 30, testResults, scenarioConfig),
-        makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 60, testResults, scenarioConfig),
-        makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 30, testResults, scenarioConfig),
-        makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 60, testResults, scenarioConfig),
-        makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 1500, testResults, scenarioConfig),
+        makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 300, testResults, scenarioConfig),
+        makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 300, testResults, scenarioConfig),
+        makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 750, testResults, scenarioConfig),
+        makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 750, testResults, scenarioConfig),
+        makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 3000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 3000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 4000, testResults, scenarioConfig),
+        makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 4500, testResults, scenarioConfig),
+        makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 4500, testResults, scenarioConfig),
+        makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 7000, testResults, scenarioConfig),
+        makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 20, testResults, scenarioConfig),
+        makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 40, testResults, scenarioConfig),
+        makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 20, testResults, scenarioConfig),
+        makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 40, testResults, scenarioConfig),
+        makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 300, testResults, scenarioConfig),
     ];
     return summaryCsvLines.join('\n') + '\n';
 }
diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json
index 742321f709..2318702449 100644
--- a/k6-tests/ncmp/config/kpi.json
+++ b/k6-tests/ncmp/config/kpi.json
@@ -6,89 +6,135 @@
   },
   "scenarios": {
     "passthrough_read_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughReadScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "0ms"
     },
     "passthrough_read_alt_id_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughReadAltIdScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "200ms"
     },
+
     "passthrough_write_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughWriteScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "400ms"
     },
     "passthrough_write_alt_id_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "passthroughWriteAltIdScenario",
-      "vus": 2,
-      "duration": "15m"
+      "rate": 5,
+      "timeUnit": "1s",
+      "duration": "15m",
+      "preAllocatedVUs": 5,
+      "startTime": "600ms"
     },
+
     "cm_handle_id_search_nofilter_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchNoFilterScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_nofilter_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchNoFilterScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "0ms"
     },
     "cm_handle_id_search_module_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchModuleScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_module_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchModuleScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "400ms"
     },
     "cm_handle_id_search_property_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchPropertyScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_property_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchPropertyScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "800ms"
     },
     "cm_handle_id_search_cpspath_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchCpsPathScenario",
-      "vus": 1,
-      "duration": "15m"
-    },
-    "cm_handle_search_cpspath_scenario": {
-      "executor": "constant-vus",
-      "exec": "cmHandleSearchCpsPathScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "1200ms"
     },
     "cm_handle_id_search_trustlevel_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleIdSearchTrustLevelScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "2s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "1600ms"
+    },
+
+    "cm_handle_search_nofilter_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchNoFilterScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "0s"
+    },
+    "cm_handle_search_module_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchModuleScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "3s"
+    },
+    "cm_handle_search_property_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchPropertyScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "6s"
+    },
+    "cm_handle_search_cpspath_scenario": {
+      "executor": "constant-arrival-rate",
+      "exec": "cmHandleSearchCpsPathScenario",
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "9s"
     },
     "cm_handle_search_trustlevel_scenario": {
-      "executor": "constant-vus",
+      "executor": "constant-arrival-rate",
       "exec": "cmHandleSearchTrustLevelScenario",
-      "vus": 1,
-      "duration": "15m"
+      "rate": 1,
+      "timeUnit": "15s",
+      "duration": "15m",
+      "preAllocatedVUs": 1,
+      "startTime": "12s"
     },
+
     "legacy_batch_produce_scenario": {
       "executor": "shared-iterations",
       "exec": "legacyBatchProduceScenario",
@@ -122,4 +168,4 @@
     "cm_search_trustlevel_duration": ["avg <= 15000"],
     "legacy_batch_read_cmhandles_per_second": ["avg >= 150"]
   }
-}
\ No newline at end of file
+}