Merge "Plot request failures for each k6 test"

3 files changed, 52 insertions, 37 deletions

diff --git a/k6-tests/ncmp/common/cmhandle-crud.js b/k6-tests/ncmp/common/cmhandle-crud.js
index 88ecdb45b8..8f53c9b9be 100644
--- a/k6-tests/ncmp/common/cmhandle-crud.js
+++ b/k6-tests/ncmp/common/cmhandle-crud.js
@@ -19,28 +19,14 @@
  */
 
 import http from 'k6/http';
-import { check, sleep } from 'k6';
-import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, MODULE_SET_TAGS, REGISTRATION_BATCH_SIZE, CONTENT_TYPE_JSON_PARAM, makeBatchOfCmHandleIds } from './utils.js';
+import { sleep } from 'k6';
+import {
+    NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES,
+    MODULE_SET_TAGS, CONTENT_TYPE_JSON_PARAM
+} from './utils.js';
 import { executeCmHandleIdSearch } from './search-base.js';
 
-export function registerAllCmHandles() {
-    forEachBatchOfCmHandles(createCmHandles);
-    waitForAllCmHandlesToBeReady();
-}
-
-export function deregisterAllCmHandles() {
-    forEachBatchOfCmHandles(deleteCmHandles);
-}
-
-function forEachBatchOfCmHandles(functionToExecute) {
-    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
-    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
-        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
-        functionToExecute(nextBatchOfCmHandleIds);
-    }
-}
-
-function createCmHandles(cmHandleIds) {
+export function createCmHandles(cmHandleIds) {
     const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`;
     const payload = {
         "dmiPlugin": DMI_PLUGIN_URL,
@@ -57,22 +43,20 @@ function createCmHandles(cmHandleIds) {
         })),
     };
     const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM);
-    check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 });
     return response;
 }
 
-function deleteCmHandles(cmHandleIds) {
+export function deleteCmHandles(cmHandleIds) {
     const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`;
     const payload = {
         "dmiPlugin": DMI_PLUGIN_URL,
         "removedCmHandles": cmHandleIds,
     };
     const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM);
-    check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 });
     return response;
 }
 
-function waitForAllCmHandlesToBeReady() {
+export function waitForAllCmHandlesToBeReady() {
     const POLLING_INTERVAL_SECONDS = 5;
     let cmHandlesReady = 0;
     do {
@@ -86,4 +70,4 @@ function getNumberOfReadyCmHandles() {
     const response = executeCmHandleIdSearch('readyCmHandles');
     const arrayOfCmHandleIds = JSON.parse(response.body);
     return arrayOfCmHandleIds.length;
-}
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index 294789f940..88da750e2b 100644
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -30,13 +30,6 @@ export const TOPIC_DATA_OPERATIONS_BATCH_READ = 'topic-data-operations-batch-rea
 export const KAFKA_BOOTSTRAP_SERVERS = ['localhost:9092'];
 export const MODULE_SET_TAGS = ['tagA','tagB','tagC',' tagD']
 
-export function recordTimeInSeconds(functionToExecute) {
-    const startTimeInMillis = Date.now();
-    functionToExecute();
-    const endTimeInMillis = Date.now();
-    const totalTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
-    return totalTimeInSeconds;
-}
 
 /**
  * Generates a batch of CM-handle IDs based on batch size and number.
@@ -69,6 +62,14 @@ export function makeCustomSummaryReport(data, options) {
         makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', data, options),
         makeSummaryCsvLine('6', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', data, options),
         makeSummaryCsvLine('7', 'Data operations batch read', 'events/second', 'data_operations_batch_read_cmhandles_per_second', data, options),
+        makeSummaryCsvLine('1x', 'Failures of Registration of CM-handles', 'number of failed requests', 'http_req_failed{group:::setup}', data, options),
+        makeSummaryCsvLine('2x', 'Failures of De-registration of CM-handles', 'number of failed requests', 'http_req_failed{group:::teardown}', data, options),
+        makeSummaryCsvLine('3x', 'Failures of CM-handle ID search with Module filter', 'number of failed requests', 'http_req_failed{scenario:id_search_module}', data, options),
+        makeSummaryCsvLine('4x', 'Failures of CM-handle search with Module filter', 'number of failed requests', 'http_req_failed{scenario:cm_search_module}', data, options),
+        makeSummaryCsvLine('5x', 'Failures of Synchronous single CM-handle pass-through read', 'number of failed requests', 'http_req_failed{scenario:passthrough_read}', data, options),
+        makeSummaryCsvLine('6x', 'Failures of Synchronous single CM-handle pass-through write', 'number of failed requests', 'http_req_failed{scenario:passthrough_write}', data, options),
+        makeSummaryCsvLine('7ax', 'Failures of Data operations batch read', 'number of failed requests', 'http_req_failed{scenario:data_operation_send_async_http_request}', data, options),
+        makeSummaryCsvLine('7bx', 'Failures of Data operations batch read consume kafka responses', 'number of failed requests', 'kafka_reader_error_count{scenario:data_operation_consume_kafka_responses}', data, options),
     ];
     return summaryCsvLines.join('\n') + '\n';
 }
diff --git a/k6-tests/ncmp/ncmp-kpi.js b/k6-tests/ncmp/ncmp-kpi.js
index f4a44dba68..013849b866 100644
--- a/k6-tests/ncmp/ncmp-kpi.js
+++ b/k6-tests/ncmp/ncmp-kpi.js
@@ -23,10 +23,14 @@ import { Gauge, Trend } from 'k6/metrics';
 import { Reader } from 'k6/x/kafka';
 import {
     TOTAL_CM_HANDLES, READ_DATA_FOR_CM_HANDLE_DELAY_MS, WRITE_DATA_FOR_CM_HANDLE_DELAY_MS,
-    makeCustomSummaryReport, recordTimeInSeconds, makeBatchOfCmHandleIds, DATA_OPERATION_READ_BATCH_SIZE,
-    TOPIC_DATA_OPERATIONS_BATCH_READ, KAFKA_BOOTSTRAP_SERVERS
+    makeCustomSummaryReport, makeBatchOfCmHandleIds, DATA_OPERATION_READ_BATCH_SIZE,
+    TOPIC_DATA_OPERATIONS_BATCH_READ, KAFKA_BOOTSTRAP_SERVERS, REGISTRATION_BATCH_SIZE
 } from './common/utils.js';
-import { registerAllCmHandles, deregisterAllCmHandles } from './common/cmhandle-crud.js';
+import {
+    createCmHandles,
+    deleteCmHandles,
+    waitForAllCmHandlesToBeReady
+} from './common/cmhandle-crud.js';
 import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js';
 import { passthroughRead, passthroughReadWithAltId, passthroughWrite, batchRead } from './common/passthrough-crud.js';
 
@@ -107,6 +111,8 @@ export const options = {
         'http_req_failed{scenario:cm_search_module}': ['rate == 0'],
         'http_req_failed{scenario:passthrough_read}': ['rate == 0'],
         'http_req_failed{scenario:passthrough_write}': ['rate == 0'],
+        'http_req_failed{group:::setup}':['rate == 0'],
+        'http_req_failed{group:::teardown}':['rate == 0'],
         'http_req_failed{scenario:data_operation_send_async_http_request}': ['rate == 0'],
         'kafka_reader_error_count{scenario:data_operation_consume_kafka_responses}': ['count == 0'],
         'data_operations_batch_read_cmhandles_per_second': ['avg >= 150'],
@@ -114,12 +120,36 @@ export const options = {
 };
 
 export function setup() {
-    const totalRegistrationTimeInSeconds = recordTimeInSeconds(registerAllCmHandles);
+    const startTimeInMillis = Date.now();
+
+    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
+    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
+        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
+        const response = createCmHandles(nextBatchOfCmHandleIds);
+        check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 });
+    }
+
+    waitForAllCmHandlesToBeReady();
+
+    const endTimeInMillis = Date.now();
+    const totalRegistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
+
     cmHandlesCreatedPerSecondGauge.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds);
 }
 
 export function teardown() {
-    const totalDeregistrationTimeInSeconds = recordTimeInSeconds(deregisterAllCmHandles);
+    const startTimeInMillis = Date.now();
+
+    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
+    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
+        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
+        const response = deleteCmHandles(nextBatchOfCmHandleIds);
+        check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 });
+    }
+
+    const endTimeInMillis = Date.now();
+    const totalDeregistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
+
     cmHandlesDeletedPerSecondGauge.add(TOTAL_CM_HANDLES / totalDeregistrationTimeInSeconds);
 }