diff options
Diffstat (limited to 'k6-tests')
| -rw-r--r-- | k6-tests/README.md | 2 | ||||
| -rw-r--r-- | k6-tests/ncmp/1-create-cmhandles.js | 44 | ||||
| -rw-r--r-- | k6-tests/ncmp/11-delete-cmhandles.js | 44 | ||||
| -rw-r--r-- | k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js | 42 | ||||
| -rw-r--r-- | k6-tests/ncmp/common/cmhandle-crud.js | 75 | ||||
| -rw-r--r-- | k6-tests/ncmp/common/passthrough-read.js | 4 | ||||
| -rw-r--r-- | k6-tests/ncmp/common/search-base.js | 28 | ||||
| -rw-r--r-- | k6-tests/ncmp/common/utils.js | 45 | ||||
| -rw-r--r-- | k6-tests/ncmp/ncmp-kpi.js (renamed from k6-tests/ncmp/10-mixed-load-test.js) | 34 | ||||
| -rwxr-xr-x | k6-tests/ncmp/run-all-tests.sh | 43 |
10 files changed, 130 insertions, 231 deletions
diff --git a/k6-tests/README.md b/k6-tests/README.md index e26b18609c..0fdebcfe9d 100644 --- a/k6-tests/README.md +++ b/k6-tests/README.md @@ -21,5 +21,5 @@ docker-compose -f docker-compose/docker-compose.yml --profile dmi-stub up To run an individual test from command line, use ```shell -k6 run ncmp/1-create-cmhandles.js +k6 run ncmp/ncmp-kpi.js ``` diff --git a/k6-tests/ncmp/1-create-cmhandles.js b/k6-tests/ncmp/1-create-cmhandles.js deleted file mode 100644 index 1c64ab011c..0000000000 --- a/k6-tests/ncmp/1-create-cmhandles.js +++ /dev/null @@ -1,44 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -import exec from 'k6/execution'; -import { TOTAL_CM_HANDLES, REGISTRATION_BATCH_SIZE, makeBatchOfCmHandleIds, makeCustomSummaryReport } from './common/utils.js'; -import { createCmHandles } from './common/cmhandle-crud.js'; - -export const options = { - vus: 1, - iterations: Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE), - thresholds: { - http_req_failed: ['rate == 0'], - http_req_duration: ['avg <= 850'], - }, -}; - -export default function () { - const batchNumber = exec.scenario.iterationInTest; - const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber); - createCmHandles(nextBatchOfCmHandleIds); -} - -export function handleSummary(data) { - return { - stdout: makeCustomSummaryReport(data, options), - }; -} diff --git a/k6-tests/ncmp/11-delete-cmhandles.js b/k6-tests/ncmp/11-delete-cmhandles.js deleted file mode 100644 index 542754b5f9..0000000000 --- a/k6-tests/ncmp/11-delete-cmhandles.js +++ /dev/null @@ -1,44 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -import exec from 'k6/execution'; -import { TOTAL_CM_HANDLES, REGISTRATION_BATCH_SIZE, makeBatchOfCmHandleIds, makeCustomSummaryReport } from './common/utils.js'; -import { deleteCmHandles } from './common/cmhandle-crud.js'; - -export const options = { - vus: 1, - iterations: Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE), - thresholds: { - http_req_failed: ['rate == 0'], - http_req_duration: ['avg <= 1050'], - }, -}; - -export default function () { - const batchNumber = exec.scenario.iterationInTest; - const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber); - deleteCmHandles(nextBatchOfCmHandleIds); -} - -export function handleSummary(data) { - return { - stdout: makeCustomSummaryReport(data, options), - }; -} diff --git a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js deleted file mode 100644 index cce85ab7cb..0000000000 --- a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js +++ /dev/null @@ -1,42 +0,0 @@ -/* - * ============LICENSE_START======================================================= - * Copyright (C) 2024 Nordix Foundation - * ================================================================================ - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * ============LICENSE_END========================================================= - */ - -import { makeCustomSummaryReport } from './common/utils.js'; -import { waitForCmHandlesToBeReady } from './common/cmhandle-crud.js'; - -export const options = { - vus: 1, - iterations: 1, - thresholds: { - http_req_failed: ['rate == 0'], - iteration_duration: ['max <= 260000'], // 4m20s - }, -}; - -export default function () { - const timeOutInSeconds = 6 * 60; - waitForCmHandlesToBeReady(timeOutInSeconds); -} - -export function handleSummary(data) { - return { - stdout: makeCustomSummaryReport(data, options), - }; -} diff --git a/k6-tests/ncmp/common/cmhandle-crud.js b/k6-tests/ncmp/common/cmhandle-crud.js index 0c3e116a19..6d5aff7fca 100644 --- a/k6-tests/ncmp/common/cmhandle-crud.js +++ b/k6-tests/ncmp/common/cmhandle-crud.js @@ -19,10 +19,28 @@ */ import http from 'k6/http'; -import { check, sleep, fail } from 'k6'; -import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES } from './utils.js'; +import { check, sleep } from 'k6'; +import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, REGISTRATION_BATCH_SIZE, CONTENT_TYPE_JSON_PARAM, makeBatchOfCmHandleIds } from './utils.js'; +import { executeCmHandleIdSearch } from './search-base.js'; -export function createCmHandles(cmHandleIds) { +export function registerAllCmHandles() { + forEachBatchOfCmHandles(createCmHandles); + waitForAllCmHandlesToBeReady(); +} + +export function deregisterAllCmHandles() { + forEachBatchOfCmHandles(deleteCmHandles); +} + +function forEachBatchOfCmHandles(functionToExecute) { + const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE); + for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) { + const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber); + functionToExecute(nextBatchOfCmHandleIds); + } +} + +function createCmHandles(cmHandleIds) { const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`; const payload = { "dmiPlugin": DMI_PLUGIN_URL, @@ -36,55 +54,34 @@ export function createCmHandles(cmHandleIds) { } })), }; - const params = { - headers: {'Content-Type': 'application/json'} - }; - const response = http.post(url, JSON.stringify(payload), params); - check(response, { - 'status equals 200': (r) => r.status === 200, - }); + const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM); + check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 }); return response; } -export function deleteCmHandles(cmHandleIds) { +function deleteCmHandles(cmHandleIds) { const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`; const payload = { "dmiPlugin": DMI_PLUGIN_URL, "removedCmHandles": cmHandleIds, }; - const params = { - headers: {'Content-Type': 'application/json'} - }; - const response = http.post(url, JSON.stringify(payload), params); - check(response, { - 'status equals 200': (r) => r.status === 200, - }); + const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM); + check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 }); return response; } -export function waitForCmHandlesToBeReady(timeOutInSeconds) { - const pollingIntervalInSeconds = 10; - const maxRetries = Math.ceil(timeOutInSeconds / pollingIntervalInSeconds); +function waitForAllCmHandlesToBeReady() { + const POLLING_INTERVAL_SECONDS = 5; let cmHandlesReady = 0; - for (let currentTry = 0; currentTry <= maxRetries; currentTry++) { - sleep(pollingIntervalInSeconds); - try { - cmHandlesReady = getNumberOfReadyCmHandles(); - } catch (error) { - console.error(`Attempt ${currentTry + 1} - Error fetching CM handles: ${error.message}`); - } - console.log(`Attempt ${currentTry + 1} - ${cmHandlesReady}/${TOTAL_CM_HANDLES} CM handles are READY`); - if (cmHandlesReady === TOTAL_CM_HANDLES) { - console.log(`All ${TOTAL_CM_HANDLES} CM handles are READY`); - return; - } - } - fail(`Timed out after ${timeOutInSeconds} seconds waiting for ${TOTAL_CM_HANDLES} CM handles to be READY`); + do { + sleep(POLLING_INTERVAL_SECONDS); + cmHandlesReady = getNumberOfReadyCmHandles(); + console.log(`${cmHandlesReady}/${TOTAL_CM_HANDLES} CM handles are READY`); + } while (cmHandlesReady < TOTAL_CM_HANDLES); } function getNumberOfReadyCmHandles() { - const endpointUrl = `${NCMP_BASE_URL}/cps/api/v2/dataspaces/NCMP-Admin/anchors/ncmp-dmi-registry/node?xpath=/dmi-registry&descendants=all`; - const jsonData = http.get(endpointUrl).json(); - const cmHandles = jsonData[0]["dmi-reg:dmi-registry"]["cm-handles"]; - return cmHandles.filter(cmhandle => cmhandle['state']['cm-handle-state'] === 'READY').length; + const response = executeCmHandleIdSearch('readyCmHandles'); + const arrayOfCmHandleIds = JSON.parse(response.body); + return arrayOfCmHandleIds.length; } diff --git a/k6-tests/ncmp/common/passthrough-read.js b/k6-tests/ncmp/common/passthrough-read.js index e4e937c9c0..89ed15af79 100644 --- a/k6-tests/ncmp/common/passthrough-read.js +++ b/k6-tests/ncmp/common/passthrough-read.js @@ -19,7 +19,6 @@ */ import http from 'k6/http'; -import { check } from 'k6'; import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js'; export function passthroughRead() { @@ -29,8 +28,5 @@ export function passthroughRead() { const datastoreName = 'ncmp-datastore:passthrough-operational'; const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${cmHandleId}/data/ds/${datastoreName}?resourceIdentifier=${resourceIdentifier}&include-descendants=${includeDescendants}` const response = http.get(url); - check(response, { - 'status equals 200': (r) => r.status === 200, - }); return response; } diff --git a/k6-tests/ncmp/common/search-base.js b/k6-tests/ncmp/common/search-base.js index 816bacac56..bc964856af 100644 --- a/k6-tests/ncmp/common/search-base.js +++ b/k6-tests/ncmp/common/search-base.js @@ -19,8 +19,7 @@ */ import http from 'k6/http'; -import { check } from 'k6'; -import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js'; +import { NCMP_BASE_URL, CONTENT_TYPE_JSON_PARAM } from './utils.js'; const SEARCH_PARAMETERS_PER_SCENARIO = { 'module': { @@ -30,30 +29,29 @@ const SEARCH_PARAMETERS_PER_SCENARIO = { 'conditionParameters': [{'moduleName': 'ietf-yang-types-1'}] } ] + }, + 'readyCmHandles': { + 'cmHandleQueryParameters': [ + { + 'conditionName': 'cmHandleWithCpsPath', + 'conditionParameters': [{'cpsPath': '//state[@cm-handle-state="READY"]'}] + } + ] } }; export function executeCmHandleSearch(scenario) { - executeSearchRequest('searches', scenario); + return executeSearchRequest('searches', scenario); } export function executeCmHandleIdSearch(scenario) { - executeSearchRequest('id-searches', scenario); + return executeSearchRequest('id-searches', scenario); } function executeSearchRequest(searchType, scenario) { const searchParameters = SEARCH_PARAMETERS_PER_SCENARIO[scenario]; const payload = JSON.stringify(searchParameters); const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${searchType}`; - const params = { - headers: {'Content-Type': 'application/json'} - }; - const response = http.post(url, payload, params); - check(response, { - 'status equals 200': (r) => r.status === 200, - }); - const responseData = JSON.parse(response.body); - check(responseData, { - 'returned list has expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES, - }); + const response = http.post(url, payload, CONTENT_TYPE_JSON_PARAM); + return response; } diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js index 55ef60a2e7..54b4c3a099 100644 --- a/k6-tests/ncmp/common/utils.js +++ b/k6-tests/ncmp/common/utils.js @@ -20,8 +20,17 @@ export const NCMP_BASE_URL = 'http://localhost:8883'; export const DMI_PLUGIN_URL = 'http://ncmp-dmi-plugin-demo-and-csit-stub:8092'; -export const TOTAL_CM_HANDLES = Number(__ENV.TOTAL_CM_HANDLES) || 20000; -export const REGISTRATION_BATCH_SIZE = Number(__ENV.REGISTRATION_BATCH_SIZE) || 100; +export const TOTAL_CM_HANDLES = 20000; +export const REGISTRATION_BATCH_SIZE = 100; +export const CONTENT_TYPE_JSON_PARAM = { headers: {'Content-Type': 'application/json'} }; + +export function recordTimeInSeconds(functionToExecute) { + const startTimeInMillis = Date.now(); + functionToExecute(); + const endTimeInMillis = Date.now(); + const totalTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0; + return totalTimeInSeconds; +} /** * Generates a batch of CM-handle IDs based on batch size and number. @@ -33,7 +42,7 @@ export function makeBatchOfCmHandleIds(batchSize, batchNumber) { const batchOfIds = []; const startIndex = 1 + batchNumber * batchSize; for (let i = 0; i < batchSize; i++) { - let cmHandleId = 'ch-' + (startIndex + i); + let cmHandleId = `ch-${startIndex + i}`; batchOfIds.push(cmHandleId); } return batchOfIds; @@ -43,22 +52,20 @@ export function getRandomCmHandleId() { return `ch-${Math.floor(Math.random() * TOTAL_CM_HANDLES) + 1}`; } -function removeBracketsAndQuotes(str) { - return str.replace(/\[|\]|"/g, ''); -} - export function makeCustomSummaryReport(data, options) { - const moduleName = `${__ENV.K6_MODULE_NAME}`; - let body = ``; - for (const condition in options.thresholds) { - let limit = JSON.stringify(options.thresholds[condition]) - limit = removeBracketsAndQuotes(limit) - let limitKey = limit.split(' ')[0] - const actual = Math.ceil(data.metrics[condition].values[limitKey]) - const result = data.metrics[condition].thresholds[limit].ok ? 'PASS' : 'FAIL' - const row = `${moduleName}\t${condition}\t${limit}\t${actual}\t${result}\n`; - body += row; - } - return body; + let summaryCsv = '#,Test Name,Unit,Limit,Actual\n'; + summaryCsv += makeSummaryCsvLine(1, 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', data, options); + summaryCsv += makeSummaryCsvLine(2, 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', data, options); + summaryCsv += makeSummaryCsvLine(3, 'CM-handle ID search with Module filter', 'milliseconds', 'http_req_duration{scenario:id_search_module}', data, options); + summaryCsv += makeSummaryCsvLine(4, 'CM-handle search with Module filter', 'milliseconds', 'http_req_duration{scenario:cm_search_module}', data, options); + summaryCsv += makeSummaryCsvLine(5, 'Synchronous single CM-handle pass-through read', 'milliseconds', 'http_req_duration{scenario:passthrough_read}', data, options); + return summaryCsv; } +function makeSummaryCsvLine(testCase, testName, unit, thresholdInK6, data, options) { + const thresholdArray = JSON.parse(JSON.stringify(options.thresholds[thresholdInK6])); + const thresholdString = thresholdArray[0]; + const [thresholdKey, thresholdOperator, thresholdValue] = thresholdString.split(/\s+/); + const actualValue = data.metrics[thresholdInK6].values[thresholdKey].toFixed(3); + return `${testCase},${testName},${unit},${thresholdValue},${actualValue}\n`; +} diff --git a/k6-tests/ncmp/10-mixed-load-test.js b/k6-tests/ncmp/ncmp-kpi.js index a6b5b01e22..91a38d99ce 100644 --- a/k6-tests/ncmp/10-mixed-load-test.js +++ b/k6-tests/ncmp/ncmp-kpi.js @@ -18,13 +18,21 @@ * ============LICENSE_END========================================================= */ -import { makeCustomSummaryReport } from './common/utils.js' +import { check } from 'k6'; +import { Gauge } from 'k6/metrics'; +import { TOTAL_CM_HANDLES, makeCustomSummaryReport, recordTimeInSeconds } from './common/utils.js'; +import { registerAllCmHandles, deregisterAllCmHandles } from './common/cmhandle-crud.js'; import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js'; import { passthroughRead } from './common/passthrough-read.js'; +let cmHandlesCreatedPerSecondGauge = new Gauge('cmhandles_created_per_second'); +let cmHandlesDeletedPerSecondGauge = new Gauge('cmhandles_deleted_per_second'); + const DURATION = '15m'; export const options = { + setupTimeout: '6m', + teardownTimeout: '6m', scenarios: { passthrough_read: { executor: 'constant-vus', @@ -45,8 +53,9 @@ export const options = { duration: DURATION, }, }, - thresholds: { + 'cmhandles_created_per_second': ['value >= 22'], + 'cmhandles_deleted_per_second': ['value >= 22'], 'http_req_failed{scenario:passthrough_read}': ['rate == 0'], 'http_req_failed{scenario:id_search_module}': ['rate == 0'], 'http_req_failed{scenario:cm_search_module}': ['rate == 0'], @@ -56,16 +65,31 @@ export const options = { }, }; +export function setup() { + const totalRegistrationTimeInSeconds = recordTimeInSeconds(registerAllCmHandles); + cmHandlesCreatedPerSecondGauge.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds); +} + +export function teardown() { + const totalDeregistrationTimeInSeconds = recordTimeInSeconds(deregisterAllCmHandles); + cmHandlesDeletedPerSecondGauge.add(TOTAL_CM_HANDLES / totalDeregistrationTimeInSeconds); +} + export function passthrough_read() { - passthroughRead(); + const response = passthroughRead(); + check(response, { 'passthrough read status equals 200': (r) => r.status === 200 }); } export function id_search_module() { - executeCmHandleIdSearch('module'); + const response = executeCmHandleIdSearch('module'); + check(response, { 'module ID search status equals 200': (r) => r.status === 200 }); + check(JSON.parse(response.body), { 'module ID search returned expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES }); } export function cm_search_module() { - executeCmHandleSearch('module'); + const response = executeCmHandleSearch('module'); + check(response, { 'module search status equals 200': (r) => r.status === 200 }); + check(JSON.parse(response.body), { 'module search returned expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES }); } export function handleSummary(data) { diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh index bf6370901f..2db32ecd76 100755 --- a/k6-tests/ncmp/run-all-tests.sh +++ b/k6-tests/ncmp/run-all-tests.sh @@ -15,29 +15,36 @@ # limitations under the License. # -ALL_TEST_SCRIPTS=( \ -1-create-cmhandles.js \ -2-wait-for-cmhandles-to-be-ready.js \ -10-mixed-load-test.js \ -11-delete-cmhandles.js \ -) +pushd "$(dirname "$0")" >/dev/null || exit 1 -pushd "$(dirname "$0")" || exit 1 +number_of_failures=0 +echo "Running K6 performance tests..." +k6 --quiet run ncmp-kpi.js > summary.csv || ((number_of_failures++)) -printf "Test Case\tCondition\tLimit\tActual\tResult\n" > summary.log +if [ -f summary.csv ]; then -number_of_failures=0 -for test_script in "${ALL_TEST_SCRIPTS[@]}"; do - echo "k6 run $test_script" - k6 --quiet run -e K6_MODULE_NAME="$test_script" "$test_script" >> summary.log || ((number_of_failures++)) -done + # Output raw CSV for plotting job + echo '-- BEGIN CSV REPORT' + cat summary.csv + echo '-- END CSV REPORT' + echo + + # Output human-readable report + echo '####################################################################################################' + echo '## K 6 P E R F O R M A N C E T E S T R E S U L T S ##' + echo '####################################################################################################' + column -t -s, summary.csv + echo + + # Clean up + rm -f summary.csv -echo '##############################################################################################################################' -echo '## K 6 P E R F O R M A N C E T E S T R E S U L T S ##' -echo '##############################################################################################################################' -awk -F$'\t' '{printf "%-40s%-50s%-20s%-10s%-6s\n", $1, $2, $3, $4, $5}' summary.log +else + echo "Error: Failed to generate summary.csv" >&2 + ((number_of_failures++)) +fi -popd || exit 1 +popd >/dev/null || exit 1 echo "NCMP TEST FAILURES: $number_of_failures" exit $number_of_failures |