diff options
Diffstat (limited to 'k6-tests/ncmp')
| -rw-r--r-- | k6-tests/ncmp/common/cmhandle-crud.js | 72 | ||||
| -rw-r--r-- | k6-tests/ncmp/common/passthrough-crud.js | 74 | ||||
| -rw-r--r-- | k6-tests/ncmp/common/search-base.js | 74 | ||||
| -rw-r--r-- | k6-tests/ncmp/common/utils.js | 115 | ||||
| -rw-r--r-- | k6-tests/ncmp/ncmp-kpi.js | 368 | ||||
| -rwxr-xr-x | k6-tests/ncmp/run-all-tests.sh | 52 |
6 files changed, 755 insertions, 0 deletions
diff --git a/k6-tests/ncmp/common/cmhandle-crud.js b/k6-tests/ncmp/common/cmhandle-crud.js new file mode 100644 index 0000000000..0fea1a44f2 --- /dev/null +++ b/k6-tests/ncmp/common/cmhandle-crud.js @@ -0,0 +1,72 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +import { sleep } from 'k6'; +import { performPostRequest, NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, MODULE_SET_TAGS +} from './utils.js'; +import { executeCmHandleIdSearch } from './search-base.js'; + +export function createCmHandles(cmHandleIds) { + const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`; + const payload = JSON.stringify(createCmHandlePayload(cmHandleIds)); + return performPostRequest(url, payload, 'createCmHandles'); +} + +export function deleteCmHandles(cmHandleIds) { + const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`; + const payload = JSON.stringify({ + "dmiPlugin": DMI_PLUGIN_URL, + "removedCmHandles": cmHandleIds, + }); + return performPostRequest(url, payload, 'deleteCmHandles'); +} + +export function waitForAllCmHandlesToBeReady() { + const POLLING_INTERVAL_SECONDS = 5; + let cmHandlesReady = 0; + do { + sleep(POLLING_INTERVAL_SECONDS); + cmHandlesReady = getNumberOfReadyCmHandles(); + console.log(`${cmHandlesReady}/${TOTAL_CM_HANDLES} CM handles are READY`); + } while (cmHandlesReady < TOTAL_CM_HANDLES); +} + +function createCmHandlePayload(cmHandleIds) { + return { + "dmiPlugin": DMI_PLUGIN_URL, + "createdCmHandles": cmHandleIds.map((cmHandleId, index) => ({ + "cmHandle": cmHandleId, + "alternateId": cmHandleId.replace('ch-', 'alt-'), + "moduleSetTag": MODULE_SET_TAGS[index % MODULE_SET_TAGS.length], + "cmHandleProperties": {"neType": "RadioNode"}, + "publicCmHandleProperties": { + "Color": "yellow", + "Size": "small", + "Shape": "cube" + } + })), + }; +} + +function getNumberOfReadyCmHandles() { + const response = executeCmHandleIdSearch('cps-path-for-ready-cm-handles'); + const arrayOfCmHandleIds = JSON.parse(response.body); + return arrayOfCmHandleIds.length; +} diff --git a/k6-tests/ncmp/common/passthrough-crud.js b/k6-tests/ncmp/common/passthrough-crud.js new file mode 100644 index 0000000000..86fcef6242 --- /dev/null +++ b/k6-tests/ncmp/common/passthrough-crud.js @@ -0,0 +1,74 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +import { randomIntBetween } from 'https://jslib.k6.io/k6-utils/1.2.0/index.js'; +import { + performPostRequest, + performGetRequest, + NCMP_BASE_URL, + LEGACY_BATCH_TOPIC_NAME, + TOTAL_CM_HANDLES, +} from './utils.js'; + +export function passthroughRead(useAlternateId) { + const cmHandleReference = getRandomCmHandleReference(useAlternateId); + const resourceIdentifier = 'my-resource-identifier'; + const datastoreName = 'ncmp-datastore:passthrough-operational'; + const includeDescendants = true; + const url = generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants); + return performGetRequest(url, 'passthroughRead'); +} + +export function passthroughWrite(useAlternateId) { + const cmHandleReference = getRandomCmHandleReference(useAlternateId); + const resourceIdentifier = 'my-resource-identifier'; + const datastoreName = 'ncmp-datastore:passthrough-running'; + const includeDescendants = false; + const url = generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants); + const payload = JSON.stringify({"neType": "BaseStation"}); + return performPostRequest(url, payload, 'passthroughWrite'); +} + +export function legacyBatchRead(cmHandleIds) { + const url = `${NCMP_BASE_URL}/ncmp/v1/data?topic=${LEGACY_BATCH_TOPIC_NAME}` + const payload = JSON.stringify({ + "operations": [ + { + "resourceIdentifier": "parent/child", + "targetIds": cmHandleIds, + "datastore": "ncmp-datastore:passthrough-operational", + "options": "(fields=schemas/schema)", + "operationId": "12", + "operation": "read" + } + ] + }); + return performPostRequest(url, payload, 'batchRead'); +} + +function getRandomCmHandleReference(useAlternateId) { + const prefix = useAlternateId ? 'alt' : 'ch'; + return `${prefix}-${randomIntBetween(1, TOTAL_CM_HANDLES)}`; +} + +function generatePassthroughUrl(cmHandleReference, datastoreName, resourceIdentifier, includeDescendants) { + const descendantsParam = includeDescendants ? `&include-descendants=${includeDescendants}` : ''; + return `${NCMP_BASE_URL}/ncmp/v1/ch/${cmHandleReference}/data/ds/${datastoreName}?resourceIdentifier=${resourceIdentifier}${descendantsParam}`; +}
\ No newline at end of file diff --git a/k6-tests/ncmp/common/search-base.js b/k6-tests/ncmp/common/search-base.js new file mode 100644 index 0000000000..af2caf71ec --- /dev/null +++ b/k6-tests/ncmp/common/search-base.js @@ -0,0 +1,74 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +import {performPostRequest, NCMP_BASE_URL} from './utils.js'; + +export function executeCmHandleSearch(scenario) { + return executeSearchRequest('searches', scenario); +} + +export function executeCmHandleIdSearch(scenario) { + return executeSearchRequest('id-searches', scenario); +} + +function executeSearchRequest(searchType, scenario) { + const searchParameters = SEARCH_PARAMETERS_PER_SCENARIO[scenario]; + const payload = JSON.stringify(searchParameters); + const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${searchType}`; + return performPostRequest(url, payload, searchType); +} + +const SEARCH_PARAMETERS_PER_SCENARIO = { + "no-filter": { + "cmHandleQueryParameters": [] + }, + "module": { + "cmHandleQueryParameters": [ + { + "conditionName": "hasAllModules", + "conditionParameters": [{"moduleName": "module100"}] + } + ] + }, + "property": { + "cmHandleQueryParameters": [ + { + "conditionName": "hasAllProperties", + "conditionParameters": [{"Color": "yellow"}] + } + ] + }, + "trust-level": { + "cmHandleQueryParameters": [ + { + "conditionName": "cmHandleWithTrustLevel", + "conditionParameters": [ {"trustLevel": "COMPLETE"} ] + } + ] + }, + "cps-path-for-ready-cm-handles": { + "cmHandleQueryParameters": [ + { + "conditionName": "cmHandleWithCpsPath", + "conditionParameters": [{"cpsPath": "//state[@cm-handle-state='READY']"}] + } + ] + } +}; diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js new file mode 100644 index 0000000000..6ddcca5fbf --- /dev/null +++ b/k6-tests/ncmp/common/utils.js @@ -0,0 +1,115 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +import http from 'k6/http'; +export const NCMP_BASE_URL = 'http://localhost:8883'; +export const DMI_PLUGIN_URL = 'http://ncmp-dmi-plugin-demo-and-csit-stub:8092'; +export const TOTAL_CM_HANDLES = 20000; +export const REGISTRATION_BATCH_SIZE = 100; +export const READ_DATA_FOR_CM_HANDLE_DELAY_MS = 300; // must have same value as in docker-compose.yml +export const WRITE_DATA_FOR_CM_HANDLE_DELAY_MS = 670; // must have same value as in docker-compose.yml +export const CONTENT_TYPE_JSON_PARAM = {'Content-Type': 'application/json'}; +export const LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE = 200; +export const LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS = 1000; +export const LEGACY_BATCH_TOPIC_NAME = 'legacy_batch_topic'; +export const KAFKA_BOOTSTRAP_SERVERS = ['localhost:9092']; +export const MODULE_SET_TAGS = ['tagA', 'tagB', 'tagC', 'tagD', 'tagE']; + + +/** + * Generates a batch of CM-handle IDs based on batch size and number. + * @param {number} batchSize - Size of each batch. + * @param {number} batchNumber - Number of the batch. + * @returns {string[]} Array of CM-handle IDs, for example ['ch-201', 'ch-202' ... 'ch-300'] + */ +export function makeBatchOfCmHandleIds(batchSize, batchNumber) { + const startIndex = 1 + batchNumber * batchSize; + return Array.from({ length: batchSize }, (_, i) => `ch-${startIndex + i}`); +} + +/** + * Helper function to perform POST requests with JSON payload and content type. + * @param {string} url - The URL to send the POST request to. + * @param {Object} payload - The JSON payload to send in the POST request. + * @param {string} metricTag - A tag for the metric endpoint. + * @returns {Object} The response from the HTTP POST request. + */ +export function performPostRequest(url, payload, metricTag) { + const metricTags = { + endpoint: metricTag + }; + + return http.post(url, payload, { + headers: CONTENT_TYPE_JSON_PARAM, + tags: metricTags + }); +} + +/** + * Helper function to perform GET requests with metric tags. + * + * This function sends an HTTP GET request to the specified URL and attaches + * a metric tag to the request, which is useful for monitoring and analytics. + * + * @param {string} url - The URL to which the GET request will be sent. + * @param {string} metricTag - A string representing the metric tag to associate with the request. + * This tag is used for monitoring and tracking the request. + * @returns {Object} The response from the HTTP GET request. The response includes the status code, + * headers, body, and other related information. + */ +export function performGetRequest(url, metricTag) { + const metricTags = { + endpoint: metricTag + }; + return http.get(url, {tags: metricTags}); +} + +export function makeCustomSummaryReport(testResults, scenarioConfig) { + const summaryCsvLines = [ + '#,Test Name,Unit,Fs Requirement,Current Expectation,Actual', + makeSummaryCsvLine('0', 'HTTP request failures for all tests', 'rate of failed requests', 'http_req_failed', 0, testResults, scenarioConfig), + makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', 70, testResults, scenarioConfig), + makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', 90, testResults, scenarioConfig), + makeSummaryCsvLine('3a', 'CM-handle ID search with No filter', 'milliseconds', 'id_search_nofilter_duration', 400, testResults, scenarioConfig), + makeSummaryCsvLine('3b', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_module_duration', 200, testResults, scenarioConfig), + makeSummaryCsvLine('3c', 'CM-handle ID search with Property filter', 'milliseconds', 'id_search_property_duration', 1300, testResults, scenarioConfig), + makeSummaryCsvLine('3d', 'CM-handle ID search with Cps Path filter', 'milliseconds', 'id_search_cpspath_duration', 1300, testResults, scenarioConfig), + makeSummaryCsvLine('3e', 'CM-handle ID search with Trust Level filter', 'milliseconds', 'id_search_trustlevel_duration', 10000, testResults, scenarioConfig), + makeSummaryCsvLine('4a', 'CM-handle search with No filter', 'milliseconds', 'cm_search_nofilter_duration', 14000, testResults, scenarioConfig), + makeSummaryCsvLine('4b', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_module_duration', 16000, testResults, scenarioConfig), + makeSummaryCsvLine('4c', 'CM-handle search with Property filter', 'milliseconds', 'cm_search_property_duration', 16000, testResults, scenarioConfig), + makeSummaryCsvLine('4d', 'CM-handle search with Cps Path filter', 'milliseconds', 'cm_search_cpspath_duration', 16000, testResults, scenarioConfig), + makeSummaryCsvLine('4e', 'CM-handle search with Trust Level filter', 'milliseconds', 'cm_search_trustlevel_duration', 26000, testResults, scenarioConfig), + makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', 30, testResults, scenarioConfig), + makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', 60, testResults, scenarioConfig), + makeSummaryCsvLine('6a', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', 30, testResults, scenarioConfig), + makeSummaryCsvLine('6b', 'NCMP overhead for Synchronous single CM-handle pass-through write with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_write_alt_id', 60, testResults, scenarioConfig), + makeSummaryCsvLine('7', 'Legacy batch read operation', 'events/second', 'legacy_batch_read_cmhandles_per_second', 1500, testResults, scenarioConfig), + ]; + return summaryCsvLines.join('\n') + '\n'; +} + +function makeSummaryCsvLine(testCase, testName, unit, measurementName, currentExpectation, testResults, scenarioConfig) { + const thresholdArray = JSON.parse(JSON.stringify(scenarioConfig.thresholds[measurementName])); + const thresholdString = thresholdArray[0]; + const [thresholdKey, thresholdOperator, thresholdValue] = thresholdString.split(/\s+/); + const actualValue = testResults.metrics[measurementName].values[thresholdKey].toFixed(3); + return `${testCase},${testName},${unit},${thresholdValue},${currentExpectation},${actualValue}`; +} diff --git a/k6-tests/ncmp/ncmp-kpi.js b/k6-tests/ncmp/ncmp-kpi.js new file mode 100644 index 0000000000..53f7629cac --- /dev/null +++ b/k6-tests/ncmp/ncmp-kpi.js @@ -0,0 +1,368 @@ +/* + * ============LICENSE_START======================================================= + * Copyright (C) 2024 Nordix Foundation + * ================================================================================ + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * ============LICENSE_END========================================================= + */ + +import { check } from 'k6'; +import { Trend } from 'k6/metrics'; +import { Reader } from 'k6/x/kafka'; +import { + TOTAL_CM_HANDLES, READ_DATA_FOR_CM_HANDLE_DELAY_MS, WRITE_DATA_FOR_CM_HANDLE_DELAY_MS, + makeCustomSummaryReport, makeBatchOfCmHandleIds, LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE, + LEGACY_BATCH_TOPIC_NAME, KAFKA_BOOTSTRAP_SERVERS, REGISTRATION_BATCH_SIZE, + LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS +} from './common/utils.js'; +import { createCmHandles, deleteCmHandles, waitForAllCmHandlesToBeReady } from './common/cmhandle-crud.js'; +import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js'; +import { passthroughRead, passthroughWrite, legacyBatchRead } from './common/passthrough-crud.js'; + +let cmHandlesCreatedPerSecondTrend = new Trend('cmhandles_created_per_second', false); +let cmHandlesDeletedPerSecondTrend = new Trend('cmhandles_deleted_per_second', false); +let passthroughReadNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_read', true); +let passthroughReadNcmpOverheadTrendWithAlternateId = new Trend('ncmp_overhead_passthrough_read_alt_id', true); +let passthroughWriteNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_write', true); +let passthroughWriteNcmpOverheadTrendWithAlternateId = new Trend('ncmp_overhead_passthrough_write_alt_id', true); +let idSearchNoFilterDurationTrend = new Trend('id_search_nofilter_duration', true); +let idSearchModuleDurationTrend = new Trend('id_search_module_duration', true); +let idSearchPropertyDurationTrend = new Trend('id_search_property_duration', true); +let idSearchCpsPathDurationTrend = new Trend('id_search_cpspath_duration', true); +let idSearchTrustLevelDurationTrend = new Trend('id_search_trustlevel_duration', true); +let cmSearchNoFilterDurationTrend = new Trend('cm_search_nofilter_duration', true); +let cmSearchModuleDurationTrend = new Trend('cm_search_module_duration', true); +let cmSearchPropertyDurationTrend = new Trend('cm_search_property_duration', true); +let cmSearchCpsPathDurationTrend = new Trend('cm_search_cpspath_duration', true); +let cmSearchTrustLevelDurationTrend = new Trend('cm_search_trustlevel_duration', true); +let legacyBatchReadCmHandlesPerSecondTrend = new Trend('legacy_batch_read_cmhandles_per_second', false); + +const legacyBatchEventReader = new Reader({ + brokers: KAFKA_BOOTSTRAP_SERVERS, + topic: LEGACY_BATCH_TOPIC_NAME, +}); + +const DURATION = '15m'; +const LEGACY_BATCH_THROUGHPUT_TEST_START_TIME = '15m30s'; + +export const options = { + setupTimeout: '20m', + teardownTimeout: '20m', + scenarios: { + passthrough_read_scenario: { + executor: 'constant-vus', + exec: 'passthroughReadScenario', + vus: 2, + duration: DURATION, + }, + passthrough_read_alt_id_scenario: { + executor: 'constant-vus', + exec: 'passthroughReadAltIdScenario', + vus: 2, + duration: DURATION, + }, + passthrough_write_scenario: { + executor: 'constant-vus', + exec: 'passthroughWriteScenario', + vus: 2, + duration: DURATION, + }, + passthrough_write_alt_id_scenario: { + executor: 'constant-vus', + exec: 'passthroughWriteAltIdScenario', + vus: 2, + duration: DURATION, + }, + cm_handle_id_search_nofilter_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchNoFilterScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_nofilter_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchNoFilterScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_id_search_module_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchModuleScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_module_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchModuleScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_id_search_property_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchPropertyScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_property_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchPropertyScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_id_search_cpspath_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchCpsPathScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_cpspath_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchCpsPathScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_id_search_trustlevel_scenario: { + executor: 'constant-vus', + exec: 'cmHandleIdSearchTrustLevelScenario', + vus: 1, + duration: DURATION, + }, + cm_handle_search_trustlevel_scenario: { + executor: 'constant-vus', + exec: 'cmHandleSearchTrustLevelScenario', + vus: 1, + duration: DURATION, + }, + legacy_batch_produce_scenario: { + executor: 'shared-iterations', + exec: 'legacyBatchProduceScenario', + vus: 2, + iterations: LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS, + startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME, + }, + legacy_batch_consume_scenario: { + executor: 'per-vu-iterations', + exec: 'legacyBatchConsumeScenario', + vus: 1, + iterations: 1, + startTime: LEGACY_BATCH_THROUGHPUT_TEST_START_TIME, + } + }, + thresholds: { + 'http_req_failed': ['rate == 0'], + 'cmhandles_created_per_second': ['avg >= 22'], + 'cmhandles_deleted_per_second': ['avg >= 22'], + 'ncmp_overhead_passthrough_read': ['avg <= 40'], + 'ncmp_overhead_passthrough_write': ['avg <= 40'], + 'ncmp_overhead_passthrough_read_alt_id': ['avg <= 40'], + 'ncmp_overhead_passthrough_write_alt_id': ['avg <= 40'], + 'id_search_nofilter_duration': ['avg <= 2000'], + 'id_search_module_duration': ['avg <= 2000'], + 'id_search_property_duration': ['avg <= 2000'], + 'id_search_cpspath_duration': ['avg <= 2000'], + 'id_search_trustlevel_duration': ['avg <= 2000'], + 'cm_search_nofilter_duration': ['avg <= 15000'], + 'cm_search_module_duration': ['avg <= 15000'], + 'cm_search_property_duration': ['avg <= 15000'], + 'cm_search_cpspath_duration': ['avg <= 15000'], + 'cm_search_trustlevel_duration': ['avg <= 15000'], + 'legacy_batch_read_cmhandles_per_second': ['avg >= 150'], + }, +}; + +export function setup() { + const startTimeInMillis = Date.now(); + + const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE); + for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) { + const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber); + const response = createCmHandles(nextBatchOfCmHandleIds); + check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 }); + } + + waitForAllCmHandlesToBeReady(); + + const endTimeInMillis = Date.now(); + const totalRegistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0; + + cmHandlesCreatedPerSecondTrend.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds); +} + +export function teardown() { + const startTimeInMillis = Date.now(); + + let DEREGISTERED_CM_HANDLES = 0 + const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE); + for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) { + const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber); + const response = deleteCmHandles(nextBatchOfCmHandleIds); + if (response.error_code === 0) { + DEREGISTERED_CM_HANDLES += REGISTRATION_BATCH_SIZE + } + check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 }); + } + + const endTimeInMillis = Date.now(); + const totalDeregistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0; + + cmHandlesDeletedPerSecondTrend.add(DEREGISTERED_CM_HANDLES / totalDeregistrationTimeInSeconds); +} + +export function passthroughReadScenario() { + const response = passthroughRead(false); + if (check(response, { 'passthrough read status equals 200': (r) => r.status === 200 })) { + const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS; + passthroughReadNcmpOverheadTrend.add(overhead); + } +} + +export function passthroughReadAltIdScenario() { + const response = passthroughRead(true); + if (check(response, { 'passthrough read with alternate Id status equals 200': (r) => r.status === 200 })) { + const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS; + passthroughReadNcmpOverheadTrendWithAlternateId.add(overhead); + } +} + +export function passthroughWriteScenario() { + const response = passthroughWrite(false); + if (check(response, { 'passthrough write status equals 201': (r) => r.status === 201 })) { + const overhead = response.timings.duration - WRITE_DATA_FOR_CM_HANDLE_DELAY_MS; + passthroughWriteNcmpOverheadTrend.add(overhead); + } +} + +export function passthroughWriteAltIdScenario() { + const response = passthroughWrite(true); + if (check(response, { 'passthrough write with alternate Id status equals 201': (r) => r.status === 201 })) { + const overhead = response.timings.duration - WRITE_DATA_FOR_CM_HANDLE_DELAY_MS; + passthroughWriteNcmpOverheadTrendWithAlternateId.add(overhead); + } +} + +export function cmHandleIdSearchNoFilterScenario() { + const response = executeCmHandleIdSearch('no-filter'); + if (check(response, { 'CM handle ID no-filter search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID no-filter search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchNoFilterDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchNoFilterScenario() { + const response = executeCmHandleSearch('no-filter'); + if (check(response, { 'CM handle no-filter search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle no-filter search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchNoFilterDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchModuleScenario() { + const response = executeCmHandleIdSearch('module'); + if (check(response, { 'CM handle ID module search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID module search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchModuleDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchModuleScenario() { + const response = executeCmHandleSearch('module'); + if (check(response, { 'CM handle module search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle module search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchModuleDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchPropertyScenario() { + const response = executeCmHandleIdSearch('property'); + if (check(response, { 'CM handle ID property search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID property search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchPropertyDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchPropertyScenario() { + const response = executeCmHandleSearch('property'); + if (check(response, { 'CM handle property search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle property search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchPropertyDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchCpsPathScenario() { + const response = executeCmHandleIdSearch('cps-path-for-ready-cm-handles'); + if (check(response, { 'CM handle ID cps path search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID cps path search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchCpsPathDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchCpsPathScenario() { + const response = executeCmHandleSearch('cps-path-for-ready-cm-handles'); + if (check(response, { 'CM handle cps path search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle cps path search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchCpsPathDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleIdSearchTrustLevelScenario() { + const response = executeCmHandleIdSearch('trust-level'); + if (check(response, { 'CM handle ID trust level search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle ID trust level search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + idSearchTrustLevelDurationTrend.add(response.timings.duration); + } +} + +export function cmHandleSearchTrustLevelScenario() { + const response = executeCmHandleSearch('trust-level'); + if (check(response, { 'CM handle trust level search status equals 200': (r) => r.status === 200 }) + && check(response, { 'CM handle trust level search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) { + cmSearchTrustLevelDurationTrend.add(response.timings.duration); + } +} + +export function legacyBatchProduceScenario() { + const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE, 0); + const response = legacyBatchRead(nextBatchOfCmHandleIds); + check(response, { 'data operation batch read status equals 200': (r) => r.status === 200 }); +} + +export function legacyBatchConsumeScenario() { + const TOTAL_MESSAGES_TO_CONSUME = LEGACY_BATCH_THROUGHPUT_TEST_NUMBER_OF_REQUESTS * LEGACY_BATCH_THROUGHPUT_TEST_BATCH_SIZE; + try { + let messagesConsumed = 0; + let startTime = Date.now(); + + while (messagesConsumed < TOTAL_MESSAGES_TO_CONSUME) { + let messages = legacyBatchEventReader.consume({ limit: 1000 }); + + if (messages.length > 0) { + messagesConsumed += messages.length; + } + } + + let endTime = Date.now(); + const timeToConsumeMessagesInSeconds = (endTime - startTime) / 1000.0; + legacyBatchReadCmHandlesPerSecondTrend.add(TOTAL_MESSAGES_TO_CONSUME / timeToConsumeMessagesInSeconds); + } catch (error) { + legacyBatchReadCmHandlesPerSecondTrend.add(0); + console.error(error); + } +} + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh new file mode 100755 index 0000000000..1fa661a472 --- /dev/null +++ b/k6-tests/ncmp/run-all-tests.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Copyright 2024 Nordix Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pushd "$(dirname "$0")" >/dev/null || exit 1 + +number_of_failures=0 +echo "Running K6 performance tests..." + +# Redirecting stderr to /dev/null to prevent large log files +k6 --quiet run ncmp-kpi.js > summary.csv 2>/dev/null || ((number_of_failures++)) + +if [ -f summary.csv ]; then + + # Output raw CSV for plotting job + echo '-- BEGIN CSV REPORT' + cat summary.csv + echo '-- END CSV REPORT' + echo + + # Output human-readable report + echo '####################################################################################################' + echo '## K 6 P E R F O R M A N C E T E S T R E S U L T S ##' + echo '####################################################################################################' + column -t -s, summary.csv + echo + + # Clean up + rm -f summary.csv + +else + echo "Error: Failed to generate summary.csv" >&2 + ((number_of_failures++)) +fi + +popd >/dev/null || exit 1 + +echo "NCMP TEST FAILURES: $number_of_failures" +exit $number_of_failures |