7 files changed, 149 insertions, 68 deletions

diff --git a/k6-tests/install-deps.sh b/k6-tests/install-deps.sh
new file mode 100755
index 0000000000..bb5deb93dd
--- /dev/null
+++ b/k6-tests/install-deps.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright 2024 Nordix Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo "---> install-deps.sh"
+echo "Installing dependencies"
+
+# Create directory for downloaded binaries.
+mkdir -p bin
+touch bin/.gitignore
+
+# Add it to the PATH, so downloaded versions will be used.
+export PATH="$(pwd)/bin:$PATH"
+
+# Download docker-compose.
+if [ ! -x bin/docker-compose ]; then
+  echo " Downloading docker-compose"
+  curl -s -L https://github.com/docker/compose/releases/download/v2.29.2/docker-compose-linux-x86_64 > bin/docker-compose
+  chmod +x bin/docker-compose
+else
+  echo " docker-compose already installed"
+fi
+docker-compose version
+
+# Download k6 with kafka extension.
+if [ ! -x bin/k6 ]; then
+  echo " Downloading k6 with kafka extension"
+  curl -s -L https://github.com/mostafa/xk6-kafka/releases/download/v0.26.0/xk6-kafka_v0.26.0_linux_amd64.tar.gz | tar -xz
+  mv dist/xk6-kafka_v0.26.0_linux_amd64 bin/k6 && rmdir dist
+  chmod +x bin/k6
+else
+  echo " k6 already installed"
+fi
+k6 --version
diff --git a/k6-tests/ncmp/common/cmhandle-crud.js b/k6-tests/ncmp/common/cmhandle-crud.js
index 88ecdb45b8..8f53c9b9be 100644
--- a/k6-tests/ncmp/common/cmhandle-crud.js
+++ b/k6-tests/ncmp/common/cmhandle-crud.js
@@ -19,28 +19,14 @@
  */
 
 import http from 'k6/http';
-import { check, sleep } from 'k6';
-import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, MODULE_SET_TAGS, REGISTRATION_BATCH_SIZE, CONTENT_TYPE_JSON_PARAM, makeBatchOfCmHandleIds } from './utils.js';
+import { sleep } from 'k6';
+import {
+    NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES,
+    MODULE_SET_TAGS, CONTENT_TYPE_JSON_PARAM
+} from './utils.js';
 import { executeCmHandleIdSearch } from './search-base.js';
 
-export function registerAllCmHandles() {
-    forEachBatchOfCmHandles(createCmHandles);
-    waitForAllCmHandlesToBeReady();
-}
-
-export function deregisterAllCmHandles() {
-    forEachBatchOfCmHandles(deleteCmHandles);
-}
-
-function forEachBatchOfCmHandles(functionToExecute) {
-    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
-    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
-        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
-        functionToExecute(nextBatchOfCmHandleIds);
-    }
-}
-
-function createCmHandles(cmHandleIds) {
+export function createCmHandles(cmHandleIds) {
     const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`;
     const payload = {
         "dmiPlugin": DMI_PLUGIN_URL,
@@ -57,22 +43,20 @@ function createCmHandles(cmHandleIds) {
         })),
     };
     const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM);
-    check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 });
     return response;
 }
 
-function deleteCmHandles(cmHandleIds) {
+export function deleteCmHandles(cmHandleIds) {
     const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`;
     const payload = {
         "dmiPlugin": DMI_PLUGIN_URL,
         "removedCmHandles": cmHandleIds,
     };
     const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM);
-    check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 });
     return response;
 }
 
-function waitForAllCmHandlesToBeReady() {
+export function waitForAllCmHandlesToBeReady() {
     const POLLING_INTERVAL_SECONDS = 5;
     let cmHandlesReady = 0;
     do {
@@ -86,4 +70,4 @@ function getNumberOfReadyCmHandles() {
     const response = executeCmHandleIdSearch('readyCmHandles');
     const arrayOfCmHandleIds = JSON.parse(response.body);
     return arrayOfCmHandleIds.length;
-}
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/common/search-base.js b/k6-tests/ncmp/common/search-base.js
index bc964856af..a7d0c925c4 100644
--- a/k6-tests/ncmp/common/search-base.js
+++ b/k6-tests/ncmp/common/search-base.js
@@ -26,7 +26,7 @@ const SEARCH_PARAMETERS_PER_SCENARIO = {
         'cmHandleQueryParameters': [
             {
                 'conditionName': 'hasAllModules',
-                'conditionParameters': [{'moduleName': 'ietf-yang-types-1'}]
+                'conditionParameters': [{'moduleName': 'ietf-yang-types'}]
             }
         ]
     },
diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index 294789f940..43036b1789 100644
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -30,13 +30,6 @@ export const TOPIC_DATA_OPERATIONS_BATCH_READ = 'topic-data-operations-batch-rea
 export const KAFKA_BOOTSTRAP_SERVERS = ['localhost:9092'];
 export const MODULE_SET_TAGS = ['tagA','tagB','tagC',' tagD']
 
-export function recordTimeInSeconds(functionToExecute) {
-    const startTimeInMillis = Date.now();
-    functionToExecute();
-    const endTimeInMillis = Date.now();
-    const totalTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
-    return totalTimeInSeconds;
-}
 
 /**
  * Generates a batch of CM-handle IDs based on batch size and number.
@@ -63,12 +56,20 @@ export function makeCustomSummaryReport(data, options) {
         '#,Test Name,Unit,Limit,Actual',
         makeSummaryCsvLine('1', 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', data, options),
         makeSummaryCsvLine('2', 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', data, options),
-        makeSummaryCsvLine('3', 'CM-handle ID search with Module filter', 'milliseconds', 'http_req_duration{scenario:id_search_module}', data, options),
-        makeSummaryCsvLine('4', 'CM-handle search with Module filter', 'milliseconds', 'http_req_duration{scenario:cm_search_module}', data, options),
+        makeSummaryCsvLine('3', 'CM-handle ID search with Module filter', 'milliseconds', 'id_search_duration', data, options),
+        makeSummaryCsvLine('4', 'CM-handle search with Module filter', 'milliseconds', 'cm_search_duration', data, options),
         makeSummaryCsvLine('5a', 'NCMP overhead for Synchronous single CM-handle pass-through read', 'milliseconds', 'ncmp_overhead_passthrough_read', data, options),
         makeSummaryCsvLine('5b', 'NCMP overhead for Synchronous single CM-handle pass-through read with alternate id', 'milliseconds', 'ncmp_overhead_passthrough_read_alt_id', data, options),
         makeSummaryCsvLine('6', 'NCMP overhead for Synchronous single CM-handle pass-through write', 'milliseconds', 'ncmp_overhead_passthrough_write', data, options),
         makeSummaryCsvLine('7', 'Data operations batch read', 'events/second', 'data_operations_batch_read_cmhandles_per_second', data, options),
+        makeSummaryCsvLine('1x', 'Failures of Registration of CM-handles', 'number of failed requests', 'http_req_failed{group:::setup}', data, options),
+        makeSummaryCsvLine('2x', 'Failures of De-registration of CM-handles', 'number of failed requests', 'http_req_failed{group:::teardown}', data, options),
+        makeSummaryCsvLine('3x', 'Failures of CM-handle ID search with Module filter', 'number of failed requests', 'http_req_failed{scenario:id_search_module}', data, options),
+        makeSummaryCsvLine('4x', 'Failures of CM-handle search with Module filter', 'number of failed requests', 'http_req_failed{scenario:cm_search_module}', data, options),
+        makeSummaryCsvLine('5x', 'Failures of Synchronous single CM-handle pass-through read', 'number of failed requests', 'http_req_failed{scenario:passthrough_read}', data, options),
+        makeSummaryCsvLine('6x', 'Failures of Synchronous single CM-handle pass-through write', 'number of failed requests', 'http_req_failed{scenario:passthrough_write}', data, options),
+        makeSummaryCsvLine('7ax', 'Failures of Data operations batch read', 'number of failed requests', 'http_req_failed{scenario:data_operation_send_async_http_request}', data, options),
+        makeSummaryCsvLine('7bx', 'Failures of Data operations batch read consume kafka responses', 'number of failed requests', 'kafka_reader_error_count{scenario:data_operation_consume_kafka_responses}', data, options),
     ];
     return summaryCsvLines.join('\n') + '\n';
 }
diff --git a/k6-tests/ncmp/ncmp-kpi.js b/k6-tests/ncmp/ncmp-kpi.js
index f4a44dba68..16c6f7d2b1 100644
--- a/k6-tests/ncmp/ncmp-kpi.js
+++ b/k6-tests/ncmp/ncmp-kpi.js
@@ -19,23 +19,29 @@
  */
 
 import { check } from 'k6';
-import { Gauge, Trend } from 'k6/metrics';
+import { Trend } from 'k6/metrics';
 import { Reader } from 'k6/x/kafka';
 import {
     TOTAL_CM_HANDLES, READ_DATA_FOR_CM_HANDLE_DELAY_MS, WRITE_DATA_FOR_CM_HANDLE_DELAY_MS,
-    makeCustomSummaryReport, recordTimeInSeconds, makeBatchOfCmHandleIds, DATA_OPERATION_READ_BATCH_SIZE,
-    TOPIC_DATA_OPERATIONS_BATCH_READ, KAFKA_BOOTSTRAP_SERVERS
+    makeCustomSummaryReport, makeBatchOfCmHandleIds, DATA_OPERATION_READ_BATCH_SIZE,
+    TOPIC_DATA_OPERATIONS_BATCH_READ, KAFKA_BOOTSTRAP_SERVERS, REGISTRATION_BATCH_SIZE
 } from './common/utils.js';
-import { registerAllCmHandles, deregisterAllCmHandles } from './common/cmhandle-crud.js';
+import {
+    createCmHandles,
+    deleteCmHandles,
+    waitForAllCmHandlesToBeReady
+} from './common/cmhandle-crud.js';
 import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js';
 import { passthroughRead, passthroughReadWithAltId, passthroughWrite, batchRead } from './common/passthrough-crud.js';
 
-let cmHandlesCreatedPerSecondGauge = new Gauge('cmhandles_created_per_second');
-let cmHandlesDeletedPerSecondGauge = new Gauge('cmhandles_deleted_per_second');
+let cmHandlesCreatedPerSecondTrend = new Trend('cmhandles_created_per_second', false);
+let cmHandlesDeletedPerSecondTrend = new Trend('cmhandles_deleted_per_second', false);
 let passthroughReadNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_read', true);
 let passthroughReadNcmpOverheadTrendWithAlternateId = new Trend('ncmp_overhead_passthrough_read_alt_id', true);
 let passthroughWriteNcmpOverheadTrend = new Trend('ncmp_overhead_passthrough_write', true);
-let dataOperationsBatchReadCmHandlePerSecondTrend = new Trend('data_operations_batch_read_cmhandles_per_second');
+let idSearchDurationTrend = new Trend('id_search_duration', true);
+let cmSearchDurationTrend = new Trend('cm_search_duration', true);
+let dataOperationsBatchReadCmHandlePerSecondTrend = new Trend('data_operations_batch_read_cmhandles_per_second', false);
 
 const reader = new Reader({
     brokers: KAFKA_BOOTSTRAP_SERVERS,
@@ -45,7 +51,7 @@ const reader = new Reader({
 const DURATION = '15m';
 
 export const options = {
-    setupTimeout: '6m',
+    setupTimeout: '8m',
     teardownTimeout: '6m',
     scenarios: {
         passthrough_read: {
@@ -96,69 +102,102 @@ export const options = {
         }
     },
     thresholds: {
-        'cmhandles_created_per_second': ['value >= 22'],
-        'cmhandles_deleted_per_second': ['value >= 22'],
+        'cmhandles_created_per_second': ['avg >= 22'],
+        'cmhandles_deleted_per_second': ['avg >= 22'],
         'ncmp_overhead_passthrough_read': ['avg <= 100'],
         'ncmp_overhead_passthrough_read_alt_id': ['avg <= 100'],
         'ncmp_overhead_passthrough_write': ['avg <= 100'],
-        'http_req_duration{scenario:id_search_module}': ['avg <= 625'],
-        'http_req_duration{scenario:cm_search_module}': ['avg <= 13000'],
+        'id_search_duration': ['avg <= 625'],
+        'cm_search_duration': ['avg <= 13000'],
+        'data_operations_batch_read_cmhandles_per_second': ['avg >= 150'],
         'http_req_failed{scenario:id_search_module}': ['rate == 0'],
         'http_req_failed{scenario:cm_search_module}': ['rate == 0'],
         'http_req_failed{scenario:passthrough_read}': ['rate == 0'],
         'http_req_failed{scenario:passthrough_write}': ['rate == 0'],
+        'http_req_failed{group:::setup}':['rate == 0'],
+        'http_req_failed{group:::teardown}':['rate == 0'],
         'http_req_failed{scenario:data_operation_send_async_http_request}': ['rate == 0'],
         'kafka_reader_error_count{scenario:data_operation_consume_kafka_responses}': ['count == 0'],
-        'data_operations_batch_read_cmhandles_per_second': ['avg >= 150'],
     },
 };
 
 export function setup() {
-    const totalRegistrationTimeInSeconds = recordTimeInSeconds(registerAllCmHandles);
-    cmHandlesCreatedPerSecondGauge.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds);
+    const startTimeInMillis = Date.now();
+
+    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
+    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
+        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
+        const response = createCmHandles(nextBatchOfCmHandleIds);
+        check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 });
+    }
+
+    waitForAllCmHandlesToBeReady();
+
+    const endTimeInMillis = Date.now();
+    const totalRegistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
+
+    cmHandlesCreatedPerSecondTrend.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds);
 }
 
 export function teardown() {
-    const totalDeregistrationTimeInSeconds = recordTimeInSeconds(deregisterAllCmHandles);
-    cmHandlesDeletedPerSecondGauge.add(TOTAL_CM_HANDLES / totalDeregistrationTimeInSeconds);
+    const startTimeInMillis = Date.now();
+
+    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
+    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
+        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
+        const response = deleteCmHandles(nextBatchOfCmHandleIds);
+        check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 });
+    }
+
+    const endTimeInMillis = Date.now();
+    const totalDeregistrationTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
+
+    cmHandlesDeletedPerSecondTrend.add(TOTAL_CM_HANDLES / totalDeregistrationTimeInSeconds);
 }
 
 export function passthrough_read() {
     const response = passthroughRead();
-    check(response, { 'passthrough read status equals 200': (r) => r.status === 200 });
-    const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS;
-    passthroughReadNcmpOverheadTrend.add(overhead);
+    if (check(response, { 'passthrough read status equals 200': (r) => r.status === 200 })) {
+        const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS;
+        passthroughReadNcmpOverheadTrend.add(overhead);
+    }
 }
 
 export function passthrough_read_alt_id() {
     const response = passthroughReadWithAltId();
-    check(response, { 'passthrough read with alternate Id status equals 200': (r) => r.status === 200 });
-    const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS;
-    passthroughReadNcmpOverheadTrendWithAlternateId.add(overhead);
+    if (check(response, { 'passthrough read with alternate Id status equals 200': (r) => r.status === 200 })) {
+        const overhead = response.timings.duration - READ_DATA_FOR_CM_HANDLE_DELAY_MS;
+        passthroughReadNcmpOverheadTrendWithAlternateId.add(overhead);
+    }
 }
 
 export function passthrough_write() {
     const response = passthroughWrite();
-    check(response, { 'passthrough write status equals 201': (r) => r.status === 201 });
-    const overhead = response.timings.duration - WRITE_DATA_FOR_CM_HANDLE_DELAY_MS;
-    passthroughWriteNcmpOverheadTrend.add(overhead);
+    if (check(response, { 'passthrough write status equals 201': (r) => r.status === 201 })) {
+        const overhead = response.timings.duration - WRITE_DATA_FOR_CM_HANDLE_DELAY_MS;
+        passthroughWriteNcmpOverheadTrend.add(overhead);
+    }
 }
 
 export function id_search_module() {
     const response = executeCmHandleIdSearch('module');
-    check(response, { 'module ID search status equals 200': (r) => r.status === 200 });
-    check(JSON.parse(response.body), { 'module ID search returned expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES });
+    if (check(response, { 'CM handle ID search status equals 200': (r) => r.status === 200 })
+     && check(response, { 'CM handle ID search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
+        idSearchDurationTrend.add(response.timings.duration);
+    }
 }
 
 export function cm_search_module() {
     const response = executeCmHandleSearch('module');
-    check(response, { 'module search status equals 200': (r) => r.status === 200 });
-    check(JSON.parse(response.body), { 'module search returned expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES });
+    if (check(response, { 'CM handle search status equals 200': (r) => r.status === 200 })
+     && check(response, { 'CM handle search returned expected CM-handles': (r) => r.json('#') === TOTAL_CM_HANDLES })) {
+        cmSearchDurationTrend.add(response.timings.duration);
+    }
 }
 
 export function data_operation_send_async_http_request() {
-    const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(DATA_OPERATION_READ_BATCH_SIZE,1);
-    const response = batchRead(nextBatchOfCmHandleIds)
+    const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(DATA_OPERATION_READ_BATCH_SIZE, 0);
+    const response = batchRead(nextBatchOfCmHandleIds);
     check(response, { 'data operation batch read status equals 200': (r) => r.status === 200 });
 }
 
diff --git a/k6-tests/run-k6-tests.sh b/k6-tests/run-k6-tests.sh
index 9b8747b1ff..b1ad38911a 100755
--- a/k6-tests/run-k6-tests.sh
+++ b/k6-tests/run-k6-tests.sh
@@ -31,6 +31,10 @@ trap on_exit EXIT
 
 pushd "$(dirname "$0")" || exit 1
 
+# Install needed dependencies.
+source install-deps.sh
+
+# Run k6 test suite.
 ./setup.sh
 ./ncmp/run-all-tests.sh
 NCMP_RESULT=$?
diff --git a/k6-tests/teardown.sh b/k6-tests/teardown.sh
index 1b4d721a23..7693dc03a4 100755
--- a/k6-tests/teardown.sh
+++ b/k6-tests/teardown.sh
@@ -19,4 +19,10 @@ echo '================================== docker info =========================='
 docker ps -a
 
 echo 'Stopping, Removing containers and volumes...'
-docker-compose -f ../docker-compose/docker-compose.yml --profile dmi-stub down --volumes
+docker_compose_cmd="docker-compose -f ../docker-compose/docker-compose.yml --profile dmi-stub down --volumes"
+# Set an environment variable CLEAN_DOCKER_IMAGES=1 to also remove docker images when done (used on jenkins job)
+if [ "${CLEAN_DOCKER_IMAGES:-0}" -eq 1 ]; then
+  $docker_compose_cmd --rmi all
+else
+  $docker_compose_cmd
+fi