diff options
Diffstat (limited to 'k6-tests')
| -rw-r--r-- | k6-tests/ncmp/common/utils.js | 31 | ||||
| -rw-r--r-- | k6-tests/ncmp/config/endurance.json | 3 | ||||
| -rw-r--r-- | k6-tests/ncmp/config/kpi.json | 159 | ||||
| -rw-r--r-- | k6-tests/ncmp/ncmp-test-runner.js | 6 |
4 files changed, 125 insertions, 74 deletions
diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index b937b4beb3..8e68edd438 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -24,6 +24,7 @@ export const testConfig = JSON.parse(open(`../config/${__ENV.TEST_PROFILE}.json` export const KAFKA_BOOTSTRAP_SERVERS = testConfig.hosts.kafkaBootstrapServer; export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl; export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl; +export const CONTAINER_UP_TIME_IN_SECONDS = testConfig.hosts.containerUpTimeInSeconds; export const LEGACY_BATCH_TOPIC_NAME = 'legacy_batch_topic'; export const TOTAL_CM_HANDLES = 20000; export const REGISTRATION_BATCH_SIZE = 100; @@ -89,21 +90,21 @@ export function makeCustomSummaryReport(testResults, scenarioConfig) { makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig), makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 50, testResults, scenarioConfig), makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 100, testResults, scenarioConfig), - makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 500, testResults, scenarioConfig), - makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 500, testResults, scenarioConfig), - makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 1200, testResults, scenarioConfig), - makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 1200, testResults, scenarioConfig), - makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 4200, testResults, scenarioConfig), - makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 8000, testResults, scenarioConfig), - makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 10000, testResults, scenarioConfig), - makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 10000, testResults, scenarioConfig), - makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 10000, testResults, scenarioConfig), - makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 13000, testResults, scenarioConfig), - makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 30, testResults, scenarioConfig), - makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 60, testResults, scenarioConfig), - makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 30, testResults, scenarioConfig), - makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 60, testResults, scenarioConfig), - makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 1500, testResults, scenarioConfig), + makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 300, testResults, scenarioConfig), + makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 300, testResults, scenarioConfig), + makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 750, testResults, scenarioConfig), + makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 750, testResults, scenarioConfig), + makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 3000, testResults, scenarioConfig), + makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 3000, testResults, scenarioConfig), + makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 4000, testResults, scenarioConfig), + makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 4500, testResults, scenarioConfig), + makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 4500, testResults, scenarioConfig), + makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 7000, testResults, scenarioConfig), + makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 20, testResults, scenarioConfig), + makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 40, testResults, scenarioConfig), + makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 20, testResults, scenarioConfig), + makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 40, testResults, scenarioConfig), + makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 300, testResults, scenarioConfig), ]; return summaryCsvLines.join('\n') + '\n'; } diff --git a/k6-tests/ncmp/config/endurance.json b/k6-tests/ncmp/config/endurance.json index d215d0a1e1..d4893a45cc 100644 --- a/k6-tests/ncmp/config/endurance.json +++ b/k6-tests/ncmp/config/endurance.json @@ -2,7 +2,8 @@ "hosts": { "ncmpBaseUrl": "http://localhost:8884", "dmiStubUrl": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092", - "kafkaBootstrapServer": "localhost:9093" + "kafkaBootstrapServer": "localhost:9093", + "containerUpTimeInSeconds": 420 }, "scenarios": { "passthrough_read_scenario": { diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json index 742321f709..eed041de85 100644 --- a/k6-tests/ncmp/config/kpi.json +++ b/k6-tests/ncmp/config/kpi.json @@ -2,93 +2,140 @@ "hosts": { "ncmpBaseUrl": "http://localhost:8883", "dmiStubUrl": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092", - "kafkaBootstrapServer": "localhost:9092" + "kafkaBootstrapServer": "localhost:9092", + "containerUpTimeInSeconds": 300 }, "scenarios": { "passthrough_read_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughReadScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "0ms" }, "passthrough_read_alt_id_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughReadAltIdScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "200ms" }, + "passthrough_write_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughWriteScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "400ms" }, "passthrough_write_alt_id_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "passthroughWriteAltIdScenario", - "vus": 2, - "duration": "15m" + "rate": 5, + "timeUnit": "1s", + "duration": "15m", + "preAllocatedVUs": 5, + "startTime": "600ms" }, + "cm_handle_id_search_nofilter_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchNoFilterScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_nofilter_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchNoFilterScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "0ms" }, "cm_handle_id_search_module_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchModuleScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_module_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchModuleScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "400ms" }, "cm_handle_id_search_property_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchPropertyScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_property_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchPropertyScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "800ms" }, "cm_handle_id_search_cpspath_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchCpsPathScenario", - "vus": 1, - "duration": "15m" - }, - "cm_handle_search_cpspath_scenario": { - "executor": "constant-vus", - "exec": "cmHandleSearchCpsPathScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "1200ms" }, "cm_handle_id_search_trustlevel_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleIdSearchTrustLevelScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "2s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "1600ms" + }, + + "cm_handle_search_nofilter_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchNoFilterScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "0s" + }, + "cm_handle_search_module_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchModuleScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "3s" + }, + "cm_handle_search_property_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchPropertyScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "6s" + }, + "cm_handle_search_cpspath_scenario": { + "executor": "constant-arrival-rate", + "exec": "cmHandleSearchCpsPathScenario", + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "9s" }, "cm_handle_search_trustlevel_scenario": { - "executor": "constant-vus", + "executor": "constant-arrival-rate", "exec": "cmHandleSearchTrustLevelScenario", - "vus": 1, - "duration": "15m" + "rate": 1, + "timeUnit": "15s", + "duration": "15m", + "preAllocatedVUs": 1, + "startTime": "12s" }, + "legacy_batch_produce_scenario": { "executor": "shared-iterations", "exec": "legacyBatchProduceScenario", @@ -122,4 +169,4 @@ "cm_search_trustlevel_duration": ["avg <= 15000"], "legacy_batch_read_cmhandles_per_second": ["avg >= 150"] } -}
\ No newline at end of file +} diff --git a/k6-tests/ncmp/ncmp-test-runner.js b/k6-tests/ncmp/ncmp-test-runner.js index e33ff1852a..9ab326c44c 100644 --- a/k6-tests/ncmp/ncmp-test-runner.js +++ b/k6-tests/ncmp/ncmp-test-runner.js @@ -18,14 +18,14 @@ * ============LICENSE_END========================================================= */ -import { check } from 'k6'; +import { check, sleep } from 'k6'; import { Trend } from 'k6/metrics'; import { Reader } from 'k6/x/kafka'; import { TOTAL_CM_HANDLES, READ_DATA_FOR_CM_HANDLE_DELAY_MS, WRITE_DATA_FOR_CM_HANDLE_DELAY_MS, makeCustomSummaryReport, makeBatchOfCmHandleIds, LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE, REGISTRATION_BATCH_SIZE, LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS, KAFKA_BOOTSTRAP_SERVERS, - LEGACY_BATCH_TOPIC_NAME, testConfig + LEGACY_BATCH_TOPIC_NAME, CONTAINER_UP_TIME_IN_SECONDS, testConfig } from './common/utils.js'; import { createCmHandles, deleteCmHandles, waitForAllCmHandlesToBeReady } from './common/cmhandle-crud.js'; import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js'; @@ -97,6 +97,8 @@ export function teardown() { const totalDeregistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0; cmHandlesDeletedPerSecondTrend.add(DEREGISTERED_CM_HANDLES / totalDeregistrationTimeInSeconds); + + sleep(CONTAINER_UP_TIME_IN_SECONDS); } export function passthroughReadScenario() { |