From 0613fbd654bacb9324e07bb2f9f4244302c48aaa Mon Sep 17 00:00:00 2001 From: "halil.cakal" Date: Mon, 27 May 2024 16:06:59 +0100 Subject: Print summary report for K6 tests - add a generic template to print summary of k6 test results - remove container logs since its around 50 mb Issue-ID: CPS-2215 Change-Id: I8971fc30b9dc8be2ed16eda6755d17a91f608b48 Signed-off-by: halil.cakal Signed-off-by: danielhanrahan Signed-off-by: halil.cakal --- k6-tests/ncmp/1-create-cmhandles.js | 34 ++++++++---- k6-tests/ncmp/10-mixed-load-test.js | 64 +++++++++++++---------- k6-tests/ncmp/11-delete-cmhandles.js | 24 +++++---- k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js | 22 +++++--- k6-tests/ncmp/3-passthrough-read.js | 22 +++++--- k6-tests/ncmp/4-id-search-no-filter.js | 9 +++- k6-tests/ncmp/5-search-no-filter.js | 9 +++- k6-tests/ncmp/6-id-search-public-property.js | 11 +++- k6-tests/ncmp/7-search-public-property.js | 11 +++- k6-tests/ncmp/8-id-search-module.js | 11 +++- k6-tests/ncmp/9-search-module.js | 11 +++- k6-tests/ncmp/run-all-tests.sh | 10 +++- k6-tests/ncmp/search-base.js | 2 +- k6-tests/ncmp/utils.js | 20 +++++++ k6-tests/teardown.sh | 3 -- 15 files changed, 183 insertions(+), 80 deletions(-) diff --git a/k6-tests/ncmp/1-create-cmhandles.js b/k6-tests/ncmp/1-create-cmhandles.js index c32cd2f03..a6f8086d2 100644 --- a/k6-tests/ncmp/1-create-cmhandles.js +++ b/k6-tests/ncmp/1-create-cmhandles.js @@ -21,36 +21,48 @@ import http from 'k6/http'; import exec from 'k6/execution'; import { check } from 'k6'; -import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds } from './utils.js'; +import { + NCMP_BASE_URL, + DMI_PLUGIN_URL, + TOTAL_CM_HANDLES, + makeBatchOfCmHandleIds, + makeCustomSummaryReport +} from './utils.js'; const BATCH_SIZE = 100; export const options = { - vus: 1, - iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE), - thresholds: { - http_req_failed: ['rate == 0'], - http_req_duration: ['avg <= 1000'], - }, + vus: 1, + iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE), + thresholds: { + http_req_failed: ['rate == 0'], + http_req_duration: ['avg <= 1000'], + }, }; -export default function() { +export default function () { const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(BATCH_SIZE, exec.scenario.iterationInTest); const payload = { "dmiPlugin": DMI_PLUGIN_URL, "createdCmHandles": nextBatchOfCmHandleIds.map(cmHandleId => ({ "cmHandle": cmHandleId, - "cmHandleProperties": { "neType": "RadioNode" }, + "cmHandleProperties": {"neType": "RadioNode"}, "publicCmHandleProperties": { "Color": "yellow", "Size": "small", "Shape": "cube" - } + } })), }; const response = http.post(NCMP_BASE_URL + '/ncmpInventory/v1/ch', JSON.stringify(payload), { - headers: { 'Content-Type': 'application/json' }, + headers: {'Content-Type': 'application/json'}, }); check(response, { 'status equals 200': (r) => r.status === 200, }); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/10-mixed-load-test.js b/k6-tests/ncmp/10-mixed-load-test.js index d13afa54a..b2c20e1da 100644 --- a/k6-tests/ncmp/10-mixed-load-test.js +++ b/k6-tests/ncmp/10-mixed-load-test.js @@ -20,37 +20,39 @@ import http from 'k6/http'; import { check } from 'k6'; -import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js' +import { NCMP_BASE_URL, getRandomCmHandleId, makeCustomSummaryReport } from './utils.js' import { searchRequest } from './search-base.js'; export const options = { - scenarios: { - passthrough_read: { - executor: 'constant-vus', - exec: 'passthrough_read', - vus: 10, - duration: '1m', + scenarios: { + passthrough_read: { + executor: 'constant-vus', + exec: 'passthrough_read', + vus: 10, + duration: '1m', + }, + id_search_module: { + executor: 'constant-vus', + exec: 'id_search_module', + vus: 5, + duration: '1m', + }, + cm_search_module: { + executor: 'constant-vus', + exec: 'cm_search_module', + vus: 5, + duration: '1m', + }, }, - id_search_module: { - executor: 'constant-vus', - exec: 'id_search_module', - vus: 5, - duration: '1m', - }, - cm_search_module: { - executor: 'constant-vus', - exec: 'cm_search_module', - vus: 5, - duration: '1m', - }, - }, - thresholds: { - http_req_failed: ['rate==0'], - 'http_req_duration{scenario:passthrough_read}': ['avg <= 2540'], // DMI delay + 40 ms - 'http_req_duration{scenario:id_search_module}': ['avg <= 200'], - 'http_req_duration{scenario:cm_search_module}': ['avg <= 35_000'], - }, + thresholds: { + 'http_req_failed{scenario:passthrough_read}': ['rate == 0'], + 'http_req_failed{scenario:id_search_module}': ['rate == 0'], + 'http_req_failed{scenario:cm_search_module}': ['rate == 0'], + 'http_req_duration{scenario:passthrough_read}': ['avg <= 2540'], // DMI delay + 40 ms + 'http_req_duration{scenario:id_search_module}': ['avg <= 200'], + 'http_req_duration{scenario:cm_search_module}': ['avg <= 35_000'], + }, }; export function passthrough_read() { @@ -68,7 +70,7 @@ export function id_search_module() { "cmHandleQueryParameters": [ { "conditionName": "hasAllModules", - "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ] + "conditionParameters": [{"moduleName": "ietf-yang-types-1"}] } ] }; @@ -80,9 +82,15 @@ export function cm_search_module() { "cmHandleQueryParameters": [ { "conditionName": "hasAllModules", - "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ] + "conditionParameters": [{"moduleName": "ietf-yang-types-1"}] } ] }; searchRequest('searches', JSON.stringify(search_filter)); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} diff --git a/k6-tests/ncmp/11-delete-cmhandles.js b/k6-tests/ncmp/11-delete-cmhandles.js index daced8ddc..dabf12cb2 100644 --- a/k6-tests/ncmp/11-delete-cmhandles.js +++ b/k6-tests/ncmp/11-delete-cmhandles.js @@ -21,28 +21,34 @@ import http from 'k6/http'; import exec from 'k6/execution'; import { check } from 'k6'; -import { NCMP_BASE_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds } from './utils.js'; +import { NCMP_BASE_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds, makeCustomSummaryReport } from './utils.js'; const BATCH_SIZE = 100; export const options = { - vus: 1, - iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE), - thresholds: { - http_req_failed: ['rate == 0'], - http_req_duration: ['avg <= 1000'], - }, + vus: 1, + iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE), + thresholds: { + http_req_failed: ['rate == 0'], + http_req_duration: ['avg <= 1000'], + }, }; -export default function() { +export default function () { const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(BATCH_SIZE, exec.scenario.iterationInTest); const payload = { "dmiPlugin": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092", "removedCmHandles": nextBatchOfCmHandleIds, }; const response = http.post(NCMP_BASE_URL + '/ncmpInventory/v1/ch', JSON.stringify(payload), { - headers: { 'Content-Type': 'application/json' }, + headers: {'Content-Type': 'application/json'}, }); check(response, { 'status equals 200': (r) => r.status === 200, }); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js index 04a5c21dd..4ecadde2b 100644 --- a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js +++ b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js @@ -20,18 +20,18 @@ import http from 'k6/http'; import { sleep, fail } from 'k6'; -import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js'; +import { makeCustomSummaryReport, NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js'; export const options = { - vus: 1, - iterations: 1, - thresholds: { - http_req_failed: ['rate == 0'], - iteration_duration: ['max <= 300_000'], // 5 minutes - }, + vus: 1, + iterations: 1, + thresholds: { + http_req_failed: ['rate == 0'], + iteration_duration: ['max <= 300_000'], // 5 minutes + }, }; -export default function() { +export default function () { waitForCmHandlesToBeReady(TOTAL_CM_HANDLES); } @@ -62,3 +62,9 @@ function getNumberOfReadyCmHandles() { const cmHandles = jsonData[0]["dmi-reg:dmi-registry"]["cm-handles"]; return cmHandles.filter(cmhandle => cmhandle['state']['cm-handle-state'] === 'READY').length; } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/3-passthrough-read.js b/k6-tests/ncmp/3-passthrough-read.js index 912cee599..0b8c58a8c 100644 --- a/k6-tests/ncmp/3-passthrough-read.js +++ b/k6-tests/ncmp/3-passthrough-read.js @@ -21,21 +21,21 @@ import http from 'k6/http'; import { check } from 'k6'; import { Trend } from "k6/metrics"; -import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js' +import { NCMP_BASE_URL, getRandomCmHandleId, makeCustomSummaryReport } from './utils.js' let ncmpOverheadTrend = new Trend("ncmp_overhead"); export const options = { - vus: 12, - duration: '30s', - thresholds: { - http_req_failed: ['rate == 0'], - ncmp_overhead: ['avg <= 40'], - }, + vus: 12, + duration: '30s', + thresholds: { + http_req_failed: ['rate == 0'], + ncmp_overhead: ['avg <= 40'], + }, }; // The function that defines VU logic. -export default function() { +export default function () { const cmHandleId = getRandomCmHandleId(); const datastoreName = 'ncmp-datastore%3Apassthrough-operational'; const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${cmHandleId}/data/ds/${datastoreName}?resourceIdentifier=x&include-descendants=true` @@ -49,3 +49,9 @@ export default function() { const overhead = response.timings.duration - dmiDelay; ncmpOverheadTrend.add(overhead); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/4-id-search-no-filter.js b/k6-tests/ncmp/4-id-search-no-filter.js index 774c60942..a99f8b5d2 100644 --- a/k6-tests/ncmp/4-id-search-no-filter.js +++ b/k6-tests/ncmp/4-id-search-no-filter.js @@ -19,6 +19,7 @@ */ import { searchRequest } from './search-base.js'; +import { makeCustomSummaryReport } from "./utils.js"; export const options = { vus: 5, @@ -30,6 +31,12 @@ export const options = { }; // The function that defines VU logic. -export default function() { +export default function () { searchRequest('id-searches', '{}') } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/5-search-no-filter.js b/k6-tests/ncmp/5-search-no-filter.js index fa08d0c40..f6a7c9971 100644 --- a/k6-tests/ncmp/5-search-no-filter.js +++ b/k6-tests/ncmp/5-search-no-filter.js @@ -19,6 +19,7 @@ */ import { searchRequest } from './search-base.js'; +import { makeCustomSummaryReport } from "./utils.js"; export const options = { vus: 5, @@ -29,6 +30,12 @@ export const options = { }, }; -export default function() { +export default function () { searchRequest('searches', '{}') } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/6-id-search-public-property.js b/k6-tests/ncmp/6-id-search-public-property.js index e8b584aec..0a06331a2 100644 --- a/k6-tests/ncmp/6-id-search-public-property.js +++ b/k6-tests/ncmp/6-id-search-public-property.js @@ -19,6 +19,7 @@ */ import { searchRequest } from './search-base.js'; +import { makeCustomSummaryReport } from "./utils.js"; export const options = { vus: 5, @@ -29,14 +30,20 @@ export const options = { }, }; -export default function() { +export default function () { const search_filter = { "cmHandleQueryParameters": [ { "conditionName": "hasAllProperties", - "conditionParameters": [ {"Color": "yellow"}, {"Size": "small"} ] + "conditionParameters": [{"Color": "yellow"}, {"Size": "small"}] } ] }; searchRequest('id-searches', JSON.stringify(search_filter)); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/7-search-public-property.js b/k6-tests/ncmp/7-search-public-property.js index 3e052dabe..9a8e339c6 100644 --- a/k6-tests/ncmp/7-search-public-property.js +++ b/k6-tests/ncmp/7-search-public-property.js @@ -19,6 +19,7 @@ */ import { searchRequest } from './search-base.js'; +import { makeCustomSummaryReport } from "./utils.js"; export const options = { vus: 5, @@ -29,14 +30,20 @@ export const options = { }, }; -export default function() { +export default function () { const search_filter = { "cmHandleQueryParameters": [ { "conditionName": "hasAllProperties", - "conditionParameters": [ {"Color": "yellow"}, {"Size": "small"} ] + "conditionParameters": [{"Color": "yellow"}, {"Size": "small"}] } ] }; searchRequest('searches', JSON.stringify(search_filter)); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/8-id-search-module.js b/k6-tests/ncmp/8-id-search-module.js index e99fbfeb2..299fe62b8 100644 --- a/k6-tests/ncmp/8-id-search-module.js +++ b/k6-tests/ncmp/8-id-search-module.js @@ -19,6 +19,7 @@ */ import { searchRequest } from './search-base.js'; +import { makeCustomSummaryReport } from "./utils.js"; export const options = { vus: 5, @@ -29,14 +30,20 @@ export const options = { }, }; -export default function() { +export default function () { const search_filter = { "cmHandleQueryParameters": [ { "conditionName": "hasAllModules", - "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ] + "conditionParameters": [{"moduleName": "ietf-yang-types-1"}] } ] }; searchRequest('id-searches', JSON.stringify(search_filter)); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/9-search-module.js b/k6-tests/ncmp/9-search-module.js index 03eb9a67a..d340bf2c7 100644 --- a/k6-tests/ncmp/9-search-module.js +++ b/k6-tests/ncmp/9-search-module.js @@ -19,6 +19,7 @@ */ import { searchRequest } from './search-base.js'; +import { makeCustomSummaryReport } from "./utils.js"; export const options = { vus: 5, @@ -29,14 +30,20 @@ export const options = { }, }; -export default function() { +export default function () { const search_filter = { "cmHandleQueryParameters": [ { "conditionName": "hasAllModules", - "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ] + "conditionParameters": [{"moduleName": "ietf-yang-types-1"}] } ] }; searchRequest('searches', JSON.stringify(search_filter)); } + +export function handleSummary(data) { + return { + stdout: makeCustomSummaryReport(data, options), + }; +} \ No newline at end of file diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh index f75cf2f70..80231b980 100755 --- a/k6-tests/ncmp/run-all-tests.sh +++ b/k6-tests/ncmp/run-all-tests.sh @@ -31,13 +31,19 @@ ALL_TEST_SCRIPTS=( \ pushd "$(dirname "$0")" || exit 1 +printf "Test Case\tCondition\tLimit\tActual\tResult\n" > summary.log + number_of_failures=0 for test_script in "${ALL_TEST_SCRIPTS[@]}"; do echo "k6 run $test_script" - k6 --quiet run "$test_script" || ((number_of_failures++)) - echo + k6 --quiet run -e K6_MODULE_NAME="$test_script" "$test_script" >> summary.log || ((number_of_failures++)) done +echo '##############################################################################################################################' +echo '## K 6 P E R F O R M A N C E T E S T R E S U L T S ##' +echo '##############################################################################################################################' +awk -F$'\t' '{printf "%-40s%-50s%-20s%-10s%-6s\n", $1, $2, $3, $4, $5}' summary.log + popd || exit 1 echo "NCMP TEST FAILURES: $number_of_failures" diff --git a/k6-tests/ncmp/search-base.js b/k6-tests/ncmp/search-base.js index 9c1b247a9..19a6a5aa6 100644 --- a/k6-tests/ncmp/search-base.js +++ b/k6-tests/ncmp/search-base.js @@ -24,7 +24,7 @@ import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js'; export function searchRequest(searchType, searchFilter) { const response = http.post(NCMP_BASE_URL + '/ncmp/v1/ch/' + searchType, searchFilter, { - headers: { 'Content-Type': 'application/json' }, + headers: {'Content-Type': 'application/json'}, }); check(response, { 'status equals 200': (r) => r.status === 200, diff --git a/k6-tests/ncmp/utils.js b/k6-tests/ncmp/utils.js index 3b61b217e..18a894052 100644 --- a/k6-tests/ncmp/utils.js +++ b/k6-tests/ncmp/utils.js @@ -39,3 +39,23 @@ export function makeBatchOfCmHandleIds(batchSize, batchNumber) { export function getRandomCmHandleId() { return 'ch-' + (Math.floor(Math.random() * TOTAL_CM_HANDLES) + 1); } + +function removeBracketsAndQuotes(str) { + return str.replace(/\[|\]|"/g, ''); +} + +export function makeCustomSummaryReport(data, options) { + const moduleName = `${__ENV.K6_MODULE_NAME}`; + let body = ``; + for (const condition in options.thresholds) { + let limit = JSON.stringify(options.thresholds[condition]) + limit = removeBracketsAndQuotes(limit) + let limitKey = limit.split(' ')[0] + const actual = Math.ceil(data.metrics[condition].values[limitKey]) + const result = data.metrics[condition].thresholds[limit].ok ? 'PASS' : 'FAIL' + const row = `${moduleName}\t${condition}\t${limit}\t${actual}\t${result}\n`; + body += row; + } + return body; +} + diff --git a/k6-tests/teardown.sh b/k6-tests/teardown.sh index 87cca7252..45422f9d1 100755 --- a/k6-tests/teardown.sh +++ b/k6-tests/teardown.sh @@ -18,9 +18,6 @@ echo '================================== docker info ==========================' docker ps -a -echo '================================== CPS-NCMP Logs ========================' -docker logs cps-and-ncmp - echo 'Stopping, Removing all running containers...' docker stop $(docker ps -aq) && docker rm $(docker ps -aq) -- cgit 1.2.3-korg