Merge "[k6] Measure CM-handle (de)registration in CM-handles/sec"

10 files changed, 130 insertions, 231 deletions

diff --git a/k6-tests/README.md b/k6-tests/README.md
index e26b18609c..0fdebcfe9d 100644
--- a/k6-tests/README.md
+++ b/k6-tests/README.md
@@ -21,5 +21,5 @@ docker-compose -f docker-compose/docker-compose.yml --profile dmi-stub up
 
 To run an individual test from command line, use
 ```shell
-k6 run ncmp/1-create-cmhandles.js
+k6 run ncmp/ncmp-kpi.js
 ```
diff --git a/k6-tests/ncmp/1-create-cmhandles.js b/k6-tests/ncmp/1-create-cmhandles.js
deleted file mode 100644
index 1c64ab011c..0000000000
--- a/k6-tests/ncmp/1-create-cmhandles.js
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- *  ============LICENSE_START=======================================================
- *  Copyright (C) 2024 Nordix Foundation
- *  ================================================================================
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- *  SPDX-License-Identifier: Apache-2.0
- *  ============LICENSE_END=========================================================
- */
-
-import exec from 'k6/execution';
-import { TOTAL_CM_HANDLES, REGISTRATION_BATCH_SIZE, makeBatchOfCmHandleIds, makeCustomSummaryReport } from './common/utils.js';
-import { createCmHandles } from './common/cmhandle-crud.js';
-
-export const options = {
-    vus: 1,
-    iterations: Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE),
-    thresholds: {
-        http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 850'],
-    },
-};
-
-export default function () {
-    const batchNumber = exec.scenario.iterationInTest;
-    const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
-    createCmHandles(nextBatchOfCmHandleIds);
-}
-
-export function handleSummary(data) {
-    return {
-        stdout: makeCustomSummaryReport(data, options),
-    };
-}
diff --git a/k6-tests/ncmp/11-delete-cmhandles.js b/k6-tests/ncmp/11-delete-cmhandles.js
deleted file mode 100644
index 542754b5f9..0000000000
--- a/k6-tests/ncmp/11-delete-cmhandles.js
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- *  ============LICENSE_START=======================================================
- *  Copyright (C) 2024 Nordix Foundation
- *  ================================================================================
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- *  SPDX-License-Identifier: Apache-2.0
- *  ============LICENSE_END=========================================================
- */
-
-import exec from 'k6/execution';
-import { TOTAL_CM_HANDLES, REGISTRATION_BATCH_SIZE, makeBatchOfCmHandleIds, makeCustomSummaryReport } from './common/utils.js';
-import { deleteCmHandles } from './common/cmhandle-crud.js';
-
-export const options = {
-    vus: 1,
-    iterations: Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE),
-    thresholds: {
-        http_req_failed: ['rate == 0'],
-        http_req_duration: ['avg <= 1050'],
-    },
-};
-
-export default function () {
-    const batchNumber = exec.scenario.iterationInTest;
-    const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
-    deleteCmHandles(nextBatchOfCmHandleIds);
-}
-
-export function handleSummary(data) {
-    return {
-        stdout: makeCustomSummaryReport(data, options),
-    };
-}
diff --git a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
deleted file mode 100644
index cce85ab7cb..0000000000
--- a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- *  ============LICENSE_START=======================================================
- *  Copyright (C) 2024 Nordix Foundation
- *  ================================================================================
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *        http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- *
- *  SPDX-License-Identifier: Apache-2.0
- *  ============LICENSE_END=========================================================
- */
-
-import { makeCustomSummaryReport } from './common/utils.js';
-import { waitForCmHandlesToBeReady } from './common/cmhandle-crud.js';
-
-export const options = {
-    vus: 1,
-    iterations: 1,
-    thresholds: {
-        http_req_failed: ['rate == 0'],
-        iteration_duration: ['max <= 260000'], // 4m20s
-    },
-};
-
-export default function () {
-    const timeOutInSeconds = 6 * 60;
-    waitForCmHandlesToBeReady(timeOutInSeconds);
-}
-
-export function handleSummary(data) {
-    return {
-        stdout: makeCustomSummaryReport(data, options),
-    };
-}
diff --git a/k6-tests/ncmp/common/cmhandle-crud.js b/k6-tests/ncmp/common/cmhandle-crud.js
index 0c3e116a19..6d5aff7fca 100644
--- a/k6-tests/ncmp/common/cmhandle-crud.js
+++ b/k6-tests/ncmp/common/cmhandle-crud.js
@@ -19,10 +19,28 @@
  */
 
 import http from 'k6/http';
-import { check, sleep, fail } from 'k6';
-import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES } from './utils.js';
+import { check, sleep } from 'k6';
+import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, REGISTRATION_BATCH_SIZE, CONTENT_TYPE_JSON_PARAM, makeBatchOfCmHandleIds } from './utils.js';
+import { executeCmHandleIdSearch } from './search-base.js';
 
-export function createCmHandles(cmHandleIds) {
+export function registerAllCmHandles() {
+    forEachBatchOfCmHandles(createCmHandles);
+    waitForAllCmHandlesToBeReady();
+}
+
+export function deregisterAllCmHandles() {
+    forEachBatchOfCmHandles(deleteCmHandles);
+}
+
+function forEachBatchOfCmHandles(functionToExecute) {
+    const TOTAL_BATCHES = Math.ceil(TOTAL_CM_HANDLES / REGISTRATION_BATCH_SIZE);
+    for (let batchNumber = 0; batchNumber < TOTAL_BATCHES; batchNumber++) {
+        const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(REGISTRATION_BATCH_SIZE, batchNumber);
+        functionToExecute(nextBatchOfCmHandleIds);
+    }
+}
+
+function createCmHandles(cmHandleIds) {
     const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`;
     const payload = {
         "dmiPlugin": DMI_PLUGIN_URL,
@@ -36,55 +54,34 @@ export function createCmHandles(cmHandleIds) {
             }
         })),
     };
-    const params = {
-        headers: {'Content-Type': 'application/json'}
-    };
-    const response = http.post(url, JSON.stringify(payload), params);
-    check(response, {
-        'status equals 200': (r) => r.status === 200,
-    });
+    const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM);
+    check(response, { 'create CM-handles status equals 200': (r) => r.status === 200 });
     return response;
 }
 
-export function deleteCmHandles(cmHandleIds) {
+function deleteCmHandles(cmHandleIds) {
     const url = `${NCMP_BASE_URL}/ncmpInventory/v1/ch`;
     const payload = {
         "dmiPlugin": DMI_PLUGIN_URL,
         "removedCmHandles": cmHandleIds,
     };
-    const params = {
-        headers: {'Content-Type': 'application/json'}
-    };
-    const response = http.post(url, JSON.stringify(payload), params);
-    check(response, {
-        'status equals 200': (r) => r.status === 200,
-    });
+    const response = http.post(url, JSON.stringify(payload), CONTENT_TYPE_JSON_PARAM);
+    check(response, { 'delete CM-handles status equals 200': (r) => r.status === 200 });
     return response;
 }
 
-export function waitForCmHandlesToBeReady(timeOutInSeconds) {
-    const pollingIntervalInSeconds = 10;
-    const maxRetries = Math.ceil(timeOutInSeconds / pollingIntervalInSeconds);
+function waitForAllCmHandlesToBeReady() {
+    const POLLING_INTERVAL_SECONDS = 5;
     let cmHandlesReady = 0;
-    for (let currentTry = 0; currentTry <= maxRetries; currentTry++) {
-        sleep(pollingIntervalInSeconds);
-        try {
-            cmHandlesReady = getNumberOfReadyCmHandles();
-        } catch (error) {
-            console.error(`Attempt ${currentTry + 1} - Error fetching CM handles: ${error.message}`);
-        }
-        console.log(`Attempt ${currentTry + 1} - ${cmHandlesReady}/${TOTAL_CM_HANDLES} CM handles are READY`);
-        if (cmHandlesReady === TOTAL_CM_HANDLES) {
-            console.log(`All ${TOTAL_CM_HANDLES} CM handles are READY`);
-            return;
-        }
-    }
-    fail(`Timed out after ${timeOutInSeconds} seconds waiting for ${TOTAL_CM_HANDLES} CM handles to be READY`);
+    do {
+        sleep(POLLING_INTERVAL_SECONDS);
+        cmHandlesReady = getNumberOfReadyCmHandles();
+        console.log(`${cmHandlesReady}/${TOTAL_CM_HANDLES} CM handles are READY`);
+    } while (cmHandlesReady < TOTAL_CM_HANDLES);
 }
 
 function getNumberOfReadyCmHandles() {
-    const endpointUrl = `${NCMP_BASE_URL}/cps/api/v2/dataspaces/NCMP-Admin/anchors/ncmp-dmi-registry/node?xpath=/dmi-registry&descendants=all`;
-    const jsonData = http.get(endpointUrl).json();
-    const cmHandles = jsonData[0]["dmi-reg:dmi-registry"]["cm-handles"];
-    return cmHandles.filter(cmhandle => cmhandle['state']['cm-handle-state'] === 'READY').length;
+    const response = executeCmHandleIdSearch('readyCmHandles');
+    const arrayOfCmHandleIds = JSON.parse(response.body);
+    return arrayOfCmHandleIds.length;
 }
diff --git a/k6-tests/ncmp/common/passthrough-read.js b/k6-tests/ncmp/common/passthrough-read.js
index e4e937c9c0..89ed15af79 100644
--- a/k6-tests/ncmp/common/passthrough-read.js
+++ b/k6-tests/ncmp/common/passthrough-read.js
@@ -19,7 +19,6 @@
  */
 
 import http from 'k6/http';
-import { check } from 'k6';
 import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js';
 
 export function passthroughRead() {
@@ -29,8 +28,5 @@ export function passthroughRead() {
     const datastoreName = 'ncmp-datastore:passthrough-operational';
     const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${cmHandleId}/data/ds/${datastoreName}?resourceIdentifier=${resourceIdentifier}&include-descendants=${includeDescendants}`
     const response = http.get(url);
-    check(response, {
-        'status equals 200': (r) => r.status === 200,
-    });
     return response;
 }
diff --git a/k6-tests/ncmp/common/search-base.js b/k6-tests/ncmp/common/search-base.js
index 816bacac56..bc964856af 100644
--- a/k6-tests/ncmp/common/search-base.js
+++ b/k6-tests/ncmp/common/search-base.js
@@ -19,8 +19,7 @@
  */
 
 import http from 'k6/http';
-import { check } from 'k6';
-import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js';
+import { NCMP_BASE_URL, CONTENT_TYPE_JSON_PARAM } from './utils.js';
 
 const SEARCH_PARAMETERS_PER_SCENARIO = {
     'module': {
@@ -30,30 +29,29 @@ const SEARCH_PARAMETERS_PER_SCENARIO = {
                 'conditionParameters': [{'moduleName': 'ietf-yang-types-1'}]
             }
         ]
+    },
+    'readyCmHandles': {
+        'cmHandleQueryParameters': [
+            {
+                'conditionName': 'cmHandleWithCpsPath',
+                'conditionParameters': [{'cpsPath': '//state[@cm-handle-state="READY"]'}]
+            }
+        ]
     }
 };
 
 export function executeCmHandleSearch(scenario) {
-    executeSearchRequest('searches', scenario);
+    return executeSearchRequest('searches', scenario);
 }
 
 export function executeCmHandleIdSearch(scenario) {
-    executeSearchRequest('id-searches', scenario);
+    return executeSearchRequest('id-searches', scenario);
 }
 
 function executeSearchRequest(searchType, scenario) {
     const searchParameters = SEARCH_PARAMETERS_PER_SCENARIO[scenario];
     const payload = JSON.stringify(searchParameters);
     const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${searchType}`;
-    const params = {
-        headers: {'Content-Type': 'application/json'}
-    };
-    const response = http.post(url, payload, params);
-    check(response, {
-        'status equals 200': (r) => r.status === 200,
-    });
-    const responseData = JSON.parse(response.body);
-    check(responseData, {
-        'returned list has expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES,
-    });
+    const response = http.post(url, payload, CONTENT_TYPE_JSON_PARAM);
+    return response;
 }
diff --git a/k6-tests/ncmp/common/utils.js b/k6-tests/ncmp/common/utils.js
index 55ef60a2e7..54b4c3a099 100644
--- a/k6-tests/ncmp/common/utils.js
+++ b/k6-tests/ncmp/common/utils.js
@@ -20,8 +20,17 @@
 
 export const NCMP_BASE_URL = 'http://localhost:8883';
 export const DMI_PLUGIN_URL = 'http://ncmp-dmi-plugin-demo-and-csit-stub:8092';
-export const TOTAL_CM_HANDLES = Number(__ENV.TOTAL_CM_HANDLES) || 20000;
-export const REGISTRATION_BATCH_SIZE = Number(__ENV.REGISTRATION_BATCH_SIZE) || 100;
+export const TOTAL_CM_HANDLES = 20000;
+export const REGISTRATION_BATCH_SIZE = 100;
+export const CONTENT_TYPE_JSON_PARAM = { headers: {'Content-Type': 'application/json'} };
+
+export function recordTimeInSeconds(functionToExecute) {
+    const startTimeInMillis = Date.now();
+    functionToExecute();
+    const endTimeInMillis = Date.now();
+    const totalTimeInSeconds = (endTimeInMillis - startTimeInMillis) / 1000.0;
+    return totalTimeInSeconds;
+}
 
 /**
  * Generates a batch of CM-handle IDs based on batch size and number.
@@ -33,7 +42,7 @@ export function makeBatchOfCmHandleIds(batchSize, batchNumber) {
     const batchOfIds = [];
     const startIndex = 1 + batchNumber * batchSize;
     for (let i = 0; i < batchSize; i++) {
-        let cmHandleId = 'ch-' + (startIndex + i);
+        let cmHandleId = `ch-${startIndex + i}`;
         batchOfIds.push(cmHandleId);
     }
     return batchOfIds;
@@ -43,22 +52,20 @@ export function getRandomCmHandleId() {
     return `ch-${Math.floor(Math.random() * TOTAL_CM_HANDLES) + 1}`;
 }
 
-function removeBracketsAndQuotes(str) {
-    return str.replace(/\[|\]|"/g, '');
-}
-
 export function makeCustomSummaryReport(data, options) {
-    const moduleName = `${__ENV.K6_MODULE_NAME}`;
-    let body = ``;
-    for (const condition in options.thresholds) {
-        let limit = JSON.stringify(options.thresholds[condition])
-        limit = removeBracketsAndQuotes(limit)
-        let limitKey = limit.split(' ')[0]
-        const actual = Math.ceil(data.metrics[condition].values[limitKey])
-        const result = data.metrics[condition].thresholds[limit].ok ? 'PASS' : 'FAIL'
-        const row = `${moduleName}\t${condition}\t${limit}\t${actual}\t${result}\n`;
-        body += row;
-    }
-    return body;
+    let summaryCsv = '#,Test Name,Unit,Limit,Actual\n';
+    summaryCsv += makeSummaryCsvLine(1, 'Registration of CM-handles', 'CM-handles/second', 'cmhandles_created_per_second', data, options);
+    summaryCsv += makeSummaryCsvLine(2, 'De-registration of CM-handles', 'CM-handles/second', 'cmhandles_deleted_per_second', data, options);
+    summaryCsv += makeSummaryCsvLine(3, 'CM-handle ID search with Module filter', 'milliseconds', 'http_req_duration{scenario:id_search_module}', data, options);
+    summaryCsv += makeSummaryCsvLine(4, 'CM-handle search with Module filter', 'milliseconds', 'http_req_duration{scenario:cm_search_module}', data, options);
+    summaryCsv += makeSummaryCsvLine(5, 'Synchronous single CM-handle pass-through read', 'milliseconds', 'http_req_duration{scenario:passthrough_read}', data, options);
+    return summaryCsv;
 }
 
+function makeSummaryCsvLine(testCase, testName, unit, thresholdInK6, data, options) {
+    const thresholdArray = JSON.parse(JSON.stringify(options.thresholds[thresholdInK6]));
+    const thresholdString = thresholdArray[0];
+    const [thresholdKey, thresholdOperator, thresholdValue] = thresholdString.split(/\s+/);
+    const actualValue = data.metrics[thresholdInK6].values[thresholdKey].toFixed(3);
+    return `${testCase},${testName},${unit},${thresholdValue},${actualValue}\n`;
+}
diff --git a/k6-tests/ncmp/10-mixed-load-test.js b/k6-tests/ncmp/ncmp-kpi.js
index a6b5b01e22..91a38d99ce 100644
--- a/k6-tests/ncmp/10-mixed-load-test.js
+++ b/k6-tests/ncmp/ncmp-kpi.js
@@ -18,13 +18,21 @@
  *  ============LICENSE_END=========================================================
  */
 
-import { makeCustomSummaryReport } from './common/utils.js'
+import { check } from 'k6';
+import { Gauge } from 'k6/metrics';
+import { TOTAL_CM_HANDLES, makeCustomSummaryReport, recordTimeInSeconds } from './common/utils.js';
+import { registerAllCmHandles, deregisterAllCmHandles } from './common/cmhandle-crud.js';
 import { executeCmHandleSearch, executeCmHandleIdSearch } from './common/search-base.js';
 import { passthroughRead } from './common/passthrough-read.js';
 
+let cmHandlesCreatedPerSecondGauge = new Gauge('cmhandles_created_per_second');
+let cmHandlesDeletedPerSecondGauge = new Gauge('cmhandles_deleted_per_second');
+
 const DURATION = '15m';
 
 export const options = {
+    setupTimeout: '6m',
+    teardownTimeout: '6m',
     scenarios: {
         passthrough_read: {
             executor: 'constant-vus',
@@ -45,8 +53,9 @@ export const options = {
             duration: DURATION,
         },
     },
-
     thresholds: {
+        'cmhandles_created_per_second': ['value >= 22'],
+        'cmhandles_deleted_per_second': ['value >= 22'],
         'http_req_failed{scenario:passthrough_read}': ['rate == 0'],
         'http_req_failed{scenario:id_search_module}': ['rate == 0'],
         'http_req_failed{scenario:cm_search_module}': ['rate == 0'],
@@ -56,16 +65,31 @@ export const options = {
     },
 };
 
+export function setup() {
+    const totalRegistrationTimeInSeconds = recordTimeInSeconds(registerAllCmHandles);
+    cmHandlesCreatedPerSecondGauge.add(TOTAL_CM_HANDLES / totalRegistrationTimeInSeconds);
+}
+
+export function teardown() {
+    const totalDeregistrationTimeInSeconds = recordTimeInSeconds(deregisterAllCmHandles);
+    cmHandlesDeletedPerSecondGauge.add(TOTAL_CM_HANDLES / totalDeregistrationTimeInSeconds);
+}
+
 export function passthrough_read() {
-    passthroughRead();
+    const response = passthroughRead();
+    check(response, { 'passthrough read status equals 200': (r) => r.status === 200 });
 }
 
 export function id_search_module() {
-    executeCmHandleIdSearch('module');
+    const response = executeCmHandleIdSearch('module');
+    check(response, { 'module ID search status equals 200': (r) => r.status === 200 });
+    check(JSON.parse(response.body), { 'module ID search returned expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES });
 }
 
 export function cm_search_module() {
-    executeCmHandleSearch('module');
+    const response = executeCmHandleSearch('module');
+    check(response, { 'module search status equals 200': (r) => r.status === 200 });
+    check(JSON.parse(response.body), { 'module search returned expected CM-handles': (arr) => arr.length === TOTAL_CM_HANDLES });
 }
 
 export function handleSummary(data) {
diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh
index bf6370901f..2db32ecd76 100755
--- a/k6-tests/ncmp/run-all-tests.sh
+++ b/k6-tests/ncmp/run-all-tests.sh
@@ -15,29 +15,36 @@
 # limitations under the License.
 #
 
-ALL_TEST_SCRIPTS=( \
-1-create-cmhandles.js \
-2-wait-for-cmhandles-to-be-ready.js \
-10-mixed-load-test.js \
-11-delete-cmhandles.js \
-)
+pushd "$(dirname "$0")" >/dev/null || exit 1
 
-pushd "$(dirname "$0")" || exit 1
+number_of_failures=0
+echo "Running K6 performance tests..."
+k6 --quiet run ncmp-kpi.js > summary.csv || ((number_of_failures++))
 
-printf "Test Case\tCondition\tLimit\tActual\tResult\n" > summary.log
+if [ -f summary.csv ]; then
 
-number_of_failures=0
-for test_script in "${ALL_TEST_SCRIPTS[@]}"; do
-  echo "k6 run $test_script"
-  k6 --quiet run -e K6_MODULE_NAME="$test_script" "$test_script" >> summary.log || ((number_of_failures++))
-done
+  # Output raw CSV for plotting job
+  echo '-- BEGIN CSV REPORT'
+  cat summary.csv
+  echo '-- END CSV REPORT'
+  echo
+
+  # Output human-readable report
+  echo '####################################################################################################'
+  echo '##                  K 6   P E R F O R M A N C E   T E S T   R E S U L T S                         ##'
+  echo '####################################################################################################'
+  column -t -s, summary.csv
+  echo
+
+  # Clean up
+  rm -f summary.csv
 
-echo '##############################################################################################################################'
-echo '##                             K 6   P E R F O R M A N C E   T E S T   R E S U L T S                                        ##'
-echo '##############################################################################################################################'
-awk -F$'\t' '{printf "%-40s%-50s%-20s%-10s%-6s\n", $1, $2, $3, $4, $5}' summary.log
+else
+  echo "Error: Failed to generate summary.csv" >&2
+  ((number_of_failures++))
+fi
 
-popd || exit 1
+popd >/dev/null || exit 1
 
 echo "NCMP TEST FAILURES: $number_of_failures"
 exit $number_of_failures