summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--k6-tests/README.md2
-rw-r--r--k6-tests/ncmp/common/utils.js6
-rw-r--r--k6-tests/ncmp/config/endurance.json98
-rw-r--r--k6-tests/ncmp/config/kpi.json121
-rw-r--r--k6-tests/ncmp/ncmp-test-runner.js (renamed from k6-tests/ncmp/ncmp-kpi.js)126
-rwxr-xr-xk6-tests/ncmp/run-all-tests.sh4
6 files changed, 218 insertions, 139 deletions
diff --git a/k6-tests/README.md b/k6-tests/README.md
index f74c9d4f7b..506e6a78a7 100644
--- a/k6-tests/README.md
+++ b/k6-tests/README.md
@@ -22,5 +22,5 @@ docker-compose -f docker-compose/docker-compose.yml --profile dmi-stub up
To run an individual test from command line, use
```shell
-k6 run ncmp/ncmp-kpi.js
+k6 run ncmp/ncmp-test-runner.js
```
diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index 45f6e96050..8abe54c0cb 100644
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -20,13 +20,11 @@
import http from 'k6/http';
-const testConfig = JSON.parse(open(`../config/${__ENV.TEST_PROFILE}.json`));
+export const testConfig = JSON.parse(open(`../config/${__ENV.TEST_PROFILE}.json`));
export const KAFKA_BOOTSTRAP_SERVERS = testConfig.hosts.kafkaBootstrapServer;
-export const LEGACY_BATCH_TOPIC_NAME = testConfig.kafka.legacyBatchTopic;
-export const DURATION = testConfig.timingConfig.testDuration;
-export const LEGACY_BATCH_THROUGHPUT_TEST_START_TIME = testConfig.timingConfig.legacyBatchThroughputTestStartTime;
export const NCMP_BASE_URL = testConfig.hosts.ncmpBaseUrl;
export const DMI_PLUGIN_URL = testConfig.hosts.dmiStubUrl;
+export const LEGACY_BATCH_TOPIC_NAME = 'legacy_batch_topic';
export const TOTAL_CM_HANDLES = 20000;
export const REGISTRATION_BATCH_SIZE = 100;
export const READ_DATA_FOR_CM_HANDLE_DELAY_MS = 300; // must have same value as in docker-compose.yml
diff --git a/k6-tests/ncmp/config/endurance.json b/k6-tests/ncmp/config/endurance.json
index c9def6ce39..d215d0a1e1 100644
--- a/k6-tests/ncmp/config/endurance.json
+++ b/k6-tests/ncmp/config/endurance.json
@@ -4,10 +4,98 @@
"dmiStubUrl": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092",
"kafkaBootstrapServer": "localhost:9093"
},
- "timingConfig": {
- "testDuration": "2h"
- },
- "kafka": {
- "legacyBatchTopic": "legacy_batch_topic"
+ "scenarios": {
+ "passthrough_read_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughReadScenario",
+ "vus": 2,
+ "duration": "2h"
+ },
+ "passthrough_read_alt_id_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughReadAltIdScenario",
+ "vus": 2,
+ "duration": "2h"
+ },
+ "passthrough_write_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughWriteScenario",
+ "vus": 2,
+ "duration": "2h"
+ },
+ "passthrough_write_alt_id_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughWriteAltIdScenario",
+ "vus": 2,
+ "duration": "2h"
+ },
+ "cm_handle_id_search_nofilter_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchNoFilterScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_search_nofilter_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchNoFilterScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_id_search_module_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchModuleScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_search_module_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchModuleScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_id_search_property_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchPropertyScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_search_property_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchPropertyScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_id_search_cpspath_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchCpsPathScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_search_cpspath_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchCpsPathScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_id_search_trustlevel_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchTrustLevelScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "cm_handle_search_trustlevel_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchTrustLevelScenario",
+ "vus": 1,
+ "duration": "2h"
+ },
+ "legacy_batch_produce_scenario": {
+ "executor": "constant-arrival-rate",
+ "exec": "legacyBatchProduceScenario",
+ "duration": "2h",
+ "rate": 1,
+ "timeUnit": "1s",
+ "preAllocatedVUs": 1
+ }
}
}
diff --git a/k6-tests/ncmp/config/kpi.json b/k6-tests/ncmp/config/kpi.json
index ad79f92d4d..742321f709 100644
--- a/k6-tests/ncmp/config/kpi.json
+++ b/k6-tests/ncmp/config/kpi.json
@@ -4,11 +4,122 @@
"dmiStubUrl": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092",
"kafkaBootstrapServer": "localhost:9092"
},
- "timingConfig": {
- "testDuration": "15m",
- "legacyBatchThroughputTestStartTime": "15m30s"
+ "scenarios": {
+ "passthrough_read_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughReadScenario",
+ "vus": 2,
+ "duration": "15m"
+ },
+ "passthrough_read_alt_id_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughReadAltIdScenario",
+ "vus": 2,
+ "duration": "15m"
+ },
+ "passthrough_write_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughWriteScenario",
+ "vus": 2,
+ "duration": "15m"
+ },
+ "passthrough_write_alt_id_scenario": {
+ "executor": "constant-vus",
+ "exec": "passthroughWriteAltIdScenario",
+ "vus": 2,
+ "duration": "15m"
+ },
+ "cm_handle_id_search_nofilter_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchNoFilterScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_search_nofilter_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchNoFilterScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_id_search_module_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchModuleScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_search_module_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchModuleScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_id_search_property_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchPropertyScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_search_property_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchPropertyScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_id_search_cpspath_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchCpsPathScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_search_cpspath_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchCpsPathScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_id_search_trustlevel_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleIdSearchTrustLevelScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "cm_handle_search_trustlevel_scenario": {
+ "executor": "constant-vus",
+ "exec": "cmHandleSearchTrustLevelScenario",
+ "vus": 1,
+ "duration": "15m"
+ },
+ "legacy_batch_produce_scenario": {
+ "executor": "shared-iterations",
+ "exec": "legacyBatchProduceScenario",
+ "vus": 2,
+ "iterations": 100
+ },
+ "legacy_batch_consume_scenario": {
+ "executor": "per-vu-iterations",
+ "exec": "legacyBatchConsumeScenario",
+ "vus": 1,
+ "iterations": 1
+ }
},
- "kafka": {
- "legacyBatchTopic": "legacy_batch_topic"
+ "thresholds": {
+ "http_req_failed": ["rate == 0"],
+ "cmhandles_created_per_second": ["avg >= 22"],
+ "cmhandles_deleted_per_second": ["avg >= 22"],
+ "ncmp_overhead_passthrough_read": ["avg <= 40"],
+ "ncmp_overhead_passthrough_write": ["avg <= 40"],
+ "ncmp_overhead_passthrough_read_alt_id": ["avg <= 40"],
+ "ncmp_overhead_passthrough_write_alt_id": ["avg <= 40"],
+ "id_search_nofilter_duration": ["avg <= 2000"],
+ "id_search_module_duration": ["avg <= 2000"],
+ "id_search_property_duration": ["avg <= 2000"],
+ "id_search_cpspath_duration": ["avg <= 2000"],
+ "id_search_trustlevel_duration": ["avg <= 2000"],
+ "cm_search_nofilter_duration": ["avg <= 15000"],
+ "cm_search_module_duration": ["avg <= 15000"],
+ "cm_search_property_duration": ["avg <= 15000"],
+ "cm_search_cpspath_duration": ["avg <= 15000"],
+ "cm_search_trustlevel_duration": ["avg <= 15000"],
+ "legacy_batch_read_cmhandles_per_second": ["avg >= 150"]
}
} \ No newline at end of file
diff --git a/k6-tests/ncmp/ncmp-kpi.js b/k6-tests/ncmp/ncmp-test-runner.js
index 20fb1e86a9..e33ff1852a 100644
--- a/k6-tests/ncmp/ncmp-kpi.js
+++ b/k6-tests/ncmp/ncmp-test-runner.js
@@ -24,8 +24,8 @@ import { Reader } from 'k6/x/kafka';
import {
TOTAL_CM_HANDLES, READ_DATA_FOR_CM_HANDLE_DELAY_MS, WRITE_DATA_FOR_CM_HANDLE_DELAY_MS,
makeCustomSummaryReport, makeBatchOfCmHandleIds, LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE,
- REGISTRATION_BATCH_SIZE, LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS, DURATION,
- LEGACY_BATCH_THROUGHPUT_TEST_START_TIME, KAFKA_BOOTSTRAP_SERVERS, LEGACY_BATCH_TOPIC_NAME
+ REGISTRATION_BATCH_SIZE, LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS, KAFKA_BOOTSTRAP_SERVERS,
+ LEGACY_BATCH_TOPIC_NAME, testConfig
} from './common/utils.js';
import { createCmHandles, deleteCmHandles, waitForAllCmHandlesToBeReady } from './common/cmhandle-crud.js';
import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js';
@@ -57,126 +57,8 @@ export const legacyBatchEventReader = new Reader({
export const options = {
setupTimeout: '20m',
teardownTimeout: '20m',
- scenarios: {
- passthrough_read_scenario: {
- executor: 'constant-vus',
- exec: 'passthroughReadScenario',
- vus: 2,
- duration: DURATION,
- },
- passthrough_read_alt_id_scenario: {
- executor: 'constant-vus',
- exec: 'passthroughReadAltIdScenario',
- vus: 2,
- duration: DURATION,
- },
- passthrough_write_scenario: {
- executor: 'constant-vus',
- exec: 'passthroughWriteScenario',
- vus: 2,
- duration: DURATION,
- },
- passthrough_write_alt_id_scenario: {
- executor: 'constant-vus',
- exec: 'passthroughWriteAltIdScenario',
- vus: 2,
- duration: DURATION,
- },
- cm_handle_id_search_nofilter_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleIdSearchNoFilterScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_search_nofilter_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleSearchNoFilterScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_id_search_module_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleIdSearchModuleScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_search_module_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleSearchModuleScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_id_search_property_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleIdSearchPropertyScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_search_property_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleSearchPropertyScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_id_search_cpspath_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleIdSearchCpsPathScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_search_cpspath_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleSearchCpsPathScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_id_search_trustlevel_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleIdSearchTrustLevelScenario',
- vus: 1,
- duration: DURATION,
- },
- cm_handle_search_trustlevel_scenario: {
- executor: 'constant-vus',
- exec: 'cmHandleSearchTrustLevelScenario',
- vus: 1,
- duration: DURATION,
- },
- legacy_batch_produce_scenario: {
- executor: 'shared-iterations',
- exec: 'legacyBatchProduceScenario',
- vus: 2,
- iterations: LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS,
- startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME,
- },
- legacy_batch_consume_scenario: {
- executor: 'per-vu-iterations',
- exec: 'legacyBatchConsumeScenario',
- vus: 1,
- iterations: 1,
- startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME,
- }
- },
- thresholds: {
- 'http_req_failed': ['rate == 0'],
- 'cmhandles_created_per_second': ['avg >= 22'],
- 'cmhandles_deleted_per_second': ['avg >= 22'],
- 'ncmp_overhead_passthrough_read': ['avg <= 40'],
- 'ncmp_overhead_passthrough_write': ['avg <= 40'],
- 'ncmp_overhead_passthrough_read_alt_id': ['avg <= 40'],
- 'ncmp_overhead_passthrough_write_alt_id': ['avg <= 40'],
- 'id_search_nofilter_duration': ['avg <= 2000'],
- 'id_search_module_duration': ['avg <= 2000'],
- 'id_search_property_duration': ['avg <= 2000'],
- 'id_search_cpspath_duration': ['avg <= 2000'],
- 'id_search_trustlevel_duration': ['avg <= 2000'],
- 'cm_search_nofilter_duration': ['avg <= 15000'],
- 'cm_search_module_duration': ['avg <= 15000'],
- 'cm_search_property_duration': ['avg <= 15000'],
- 'cm_search_cpspath_duration': ['avg <= 15000'],
- 'cm_search_trustlevel_duration': ['avg <= 15000'],
- 'legacy_batch_read_cmhandles_per_second': ['avg >= 150'],
- },
+ scenarios: testConfig.scenarios,
+ thresholds: testConfig.thresholds,
};
export function setup() {
diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh
index 3457903c01..3769570ead 100755
--- a/k6-tests/ncmp/run-all-tests.sh
+++ b/k6-tests/ncmp/run-all-tests.sh
@@ -20,9 +20,9 @@ pushd "$(dirname "$0")" >/dev/null || exit 1
number_of_failures=0
testProfile=$1
summaryFile="${testProfile}Summary.csv"
-
echo "Running $testProfile performance tests..."
-k6 run ncmp-kpi.js --quiet -e TEST_PROFILE="$testProfile" > "$summaryFile" 2>/dev/null || ((number_of_failures++))
+
+k6 run ncmp-test-runner.js --quiet -e TEST_PROFILE="$testProfile" > "$summaryFile" 2>/dev/null || ((number_of_failures++))
if [ -f "$summaryFile" ]; then