Print summary report for K6 tests

- add a generic template to print summary of k6 test results
- remove container logs since its around 50 mb

Issue-ID: CPS-2215
Change-Id: I8971fc30b9dc8be2ed16eda6755d17a91f608b48
Signed-off-by: halil.cakal <halil.cakal@est.tech>
Signed-off-by: danielhanrahan <daniel.hanrahan@est.tech>
Signed-off-by: halil.cakal <halil.cakal@est.tech>

15 files changed, 183 insertions, 80 deletions

diff --git a/k6-tests/ncmp/1-create-cmhandles.js b/k6-tests/ncmp/1-create-cmhandles.js
index c32cd2f03d..a6f8086d20 100644
--- a/k6-tests/ncmp/1-create-cmhandles.js
+++ b/k6-tests/ncmp/1-create-cmhandles.js
@@ -21,36 +21,48 @@
 import http from 'k6/http';
 import exec from 'k6/execution';
 import { check } from 'k6';
-import { NCMP_BASE_URL, DMI_PLUGIN_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds } from './utils.js';
+import {
+    NCMP_BASE_URL,
+    DMI_PLUGIN_URL,
+    TOTAL_CM_HANDLES,
+    makeBatchOfCmHandleIds,
+    makeCustomSummaryReport
+} from './utils.js';
 
 const BATCH_SIZE = 100;
 export const options = {
-  vus: 1,
-  iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
-  thresholds: {
-    http_req_failed: ['rate == 0'],
-    http_req_duration: ['avg <= 1000'],
-  },
+    vus: 1,
+    iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
+    thresholds: {
+        http_req_failed: ['rate == 0'],
+        http_req_duration: ['avg <= 1000'],
+    },
 };
 
-export default function() {
+export default function () {
     const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(BATCH_SIZE, exec.scenario.iterationInTest);
     const payload = {
         "dmiPlugin": DMI_PLUGIN_URL,
         "createdCmHandles": nextBatchOfCmHandleIds.map(cmHandleId => ({
             "cmHandle": cmHandleId,
-            "cmHandleProperties": { "neType": "RadioNode" },
+            "cmHandleProperties": {"neType": "RadioNode"},
             "publicCmHandleProperties": {
                 "Color": "yellow",
                 "Size": "small",
                 "Shape": "cube"
-           }
+            }
         })),
     };
     const response = http.post(NCMP_BASE_URL + '/ncmpInventory/v1/ch', JSON.stringify(payload), {
-        headers: { 'Content-Type': 'application/json' },
+        headers: {'Content-Type': 'application/json'},
     });
     check(response, {
         'status equals 200': (r) => r.status === 200,
     });
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/10-mixed-load-test.js b/k6-tests/ncmp/10-mixed-load-test.js
index d13afa54ab..b2c20e1da5 100644
--- a/k6-tests/ncmp/10-mixed-load-test.js
+++ b/k6-tests/ncmp/10-mixed-load-test.js
@@ -20,37 +20,39 @@
 
 import http from 'k6/http';
 import { check } from 'k6';
-import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js'
+import { NCMP_BASE_URL, getRandomCmHandleId, makeCustomSummaryReport } from './utils.js'
 import { searchRequest } from './search-base.js';
 
 export const options = {
-  scenarios: {
-    passthrough_read: {
-      executor: 'constant-vus',
-      exec: 'passthrough_read',
-      vus: 10,
-      duration: '1m',
+    scenarios: {
+        passthrough_read: {
+            executor: 'constant-vus',
+            exec: 'passthrough_read',
+            vus: 10,
+            duration: '1m',
+        },
+        id_search_module: {
+            executor: 'constant-vus',
+            exec: 'id_search_module',
+            vus: 5,
+            duration: '1m',
+        },
+        cm_search_module: {
+            executor: 'constant-vus',
+            exec: 'cm_search_module',
+            vus: 5,
+            duration: '1m',
+        },
     },
-    id_search_module: {
-      executor: 'constant-vus',
-      exec: 'id_search_module',
-      vus: 5,
-      duration: '1m',
-    },
-    cm_search_module: {
-      executor: 'constant-vus',
-      exec: 'cm_search_module',
-      vus: 5,
-      duration: '1m',
-    },
-  },
 
-  thresholds: {
-    http_req_failed: ['rate==0'],
-    'http_req_duration{scenario:passthrough_read}': ['avg <= 2540'], // DMI delay + 40 ms
-    'http_req_duration{scenario:id_search_module}': ['avg <= 200'],
-    'http_req_duration{scenario:cm_search_module}': ['avg <= 35_000'],
-  },
+    thresholds: {
+        'http_req_failed{scenario:passthrough_read}': ['rate == 0'],
+        'http_req_failed{scenario:id_search_module}': ['rate == 0'],
+        'http_req_failed{scenario:cm_search_module}': ['rate == 0'],
+        'http_req_duration{scenario:passthrough_read}': ['avg <= 2540'], // DMI delay + 40 ms
+        'http_req_duration{scenario:id_search_module}': ['avg <= 200'],
+        'http_req_duration{scenario:cm_search_module}': ['avg <= 35_000'],
+    },
 };
 
 export function passthrough_read() {
@@ -68,7 +70,7 @@ export function id_search_module() {
         "cmHandleQueryParameters": [
             {
                 "conditionName": "hasAllModules",
-                "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+                "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
             }
         ]
     };
@@ -80,9 +82,15 @@ export function cm_search_module() {
         "cmHandleQueryParameters": [
             {
                 "conditionName": "hasAllModules",
-                "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+                "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
             }
         ]
     };
     searchRequest('searches', JSON.stringify(search_filter));
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
diff --git a/k6-tests/ncmp/11-delete-cmhandles.js b/k6-tests/ncmp/11-delete-cmhandles.js
index daced8ddc8..dabf12cb26 100644
--- a/k6-tests/ncmp/11-delete-cmhandles.js
+++ b/k6-tests/ncmp/11-delete-cmhandles.js
@@ -21,28 +21,34 @@
 import http from 'k6/http';
 import exec from 'k6/execution';
 import { check } from 'k6';
-import { NCMP_BASE_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds } from './utils.js';
+import { NCMP_BASE_URL, TOTAL_CM_HANDLES, makeBatchOfCmHandleIds, makeCustomSummaryReport } from './utils.js';
 
 const BATCH_SIZE = 100;
 export const options = {
-  vus: 1,
-  iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
-  thresholds: {
-    http_req_failed: ['rate == 0'],
-    http_req_duration: ['avg <= 1000'],
-  },
+    vus: 1,
+    iterations: Math.ceil(TOTAL_CM_HANDLES / BATCH_SIZE),
+    thresholds: {
+        http_req_failed: ['rate == 0'],
+        http_req_duration: ['avg <= 1000'],
+    },
 };
 
-export default function() {
+export default function () {
     const nextBatchOfCmHandleIds = makeBatchOfCmHandleIds(BATCH_SIZE, exec.scenario.iterationInTest);
     const payload = {
         "dmiPlugin": "http://ncmp-dmi-plugin-demo-and-csit-stub:8092",
         "removedCmHandles": nextBatchOfCmHandleIds,
     };
     const response = http.post(NCMP_BASE_URL + '/ncmpInventory/v1/ch', JSON.stringify(payload), {
-        headers: { 'Content-Type': 'application/json' },
+        headers: {'Content-Type': 'application/json'},
     });
     check(response, {
         'status equals 200': (r) => r.status === 200,
     });
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
index 04a5c21dd6..4ecadde2b4 100644
--- a/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
+++ b/k6-tests/ncmp/2-wait-for-cmhandles-to-be-ready.js
@@ -20,18 +20,18 @@
 
 import http from 'k6/http';
 import { sleep, fail } from 'k6';
-import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js';
+import { makeCustomSummaryReport, NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js';
 
 export const options = {
-  vus: 1,
-  iterations: 1,
-  thresholds: {
-    http_req_failed: ['rate == 0'],
-    iteration_duration: ['max <= 300_000'], // 5 minutes
-  },
+    vus: 1,
+    iterations: 1,
+    thresholds: {
+        http_req_failed: ['rate == 0'],
+        iteration_duration: ['max <= 300_000'], // 5 minutes
+    },
 };
 
-export default function() {
+export default function () {
     waitForCmHandlesToBeReady(TOTAL_CM_HANDLES);
 }
 
@@ -62,3 +62,9 @@ function getNumberOfReadyCmHandles() {
     const cmHandles = jsonData[0]["dmi-reg:dmi-registry"]["cm-handles"];
     return cmHandles.filter(cmhandle => cmhandle['state']['cm-handle-state'] === 'READY').length;
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/3-passthrough-read.js b/k6-tests/ncmp/3-passthrough-read.js
index 912cee5997..0b8c58a8ca 100644
--- a/k6-tests/ncmp/3-passthrough-read.js
+++ b/k6-tests/ncmp/3-passthrough-read.js
@@ -21,21 +21,21 @@
 import http from 'k6/http';
 import { check } from 'k6';
 import { Trend } from "k6/metrics";
-import { NCMP_BASE_URL, getRandomCmHandleId } from './utils.js'
+import { NCMP_BASE_URL, getRandomCmHandleId, makeCustomSummaryReport } from './utils.js'
 
 let ncmpOverheadTrend = new Trend("ncmp_overhead");
 
 export const options = {
-  vus: 12,
-  duration: '30s',
-  thresholds: {
-    http_req_failed: ['rate == 0'],
-    ncmp_overhead: ['avg <= 40'],
-  },
+    vus: 12,
+    duration: '30s',
+    thresholds: {
+        http_req_failed: ['rate == 0'],
+        ncmp_overhead: ['avg <= 40'],
+    },
 };
 
 // The function that defines VU logic.
-export default function() {
+export default function () {
     const cmHandleId = getRandomCmHandleId();
     const datastoreName = 'ncmp-datastore%3Apassthrough-operational';
     const url = `${NCMP_BASE_URL}/ncmp/v1/ch/${cmHandleId}/data/ds/${datastoreName}?resourceIdentifier=x&include-descendants=true`
@@ -49,3 +49,9 @@ export default function() {
     const overhead = response.timings.duration - dmiDelay;
     ncmpOverheadTrend.add(overhead);
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/4-id-search-no-filter.js b/k6-tests/ncmp/4-id-search-no-filter.js
index 774c609423..a99f8b5d20 100644
--- a/k6-tests/ncmp/4-id-search-no-filter.js
+++ b/k6-tests/ncmp/4-id-search-no-filter.js
@@ -19,6 +19,7 @@
  */
 
 import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
     vus: 5,
@@ -30,6 +31,12 @@ export const options = {
 };
 
 // The function that defines VU logic.
-export default function() {
+export default function () {
     searchRequest('id-searches', '{}')
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/5-search-no-filter.js b/k6-tests/ncmp/5-search-no-filter.js
index fa08d0c40c..f6a7c99719 100644
--- a/k6-tests/ncmp/5-search-no-filter.js
+++ b/k6-tests/ncmp/5-search-no-filter.js
@@ -19,6 +19,7 @@
  */
 
 import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
     vus: 5,
@@ -29,6 +30,12 @@ export const options = {
     },
 };
 
-export default function() {
+export default function () {
     searchRequest('searches', '{}')
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/6-id-search-public-property.js b/k6-tests/ncmp/6-id-search-public-property.js
index e8b584aecb..0a06331a2d 100644
--- a/k6-tests/ncmp/6-id-search-public-property.js
+++ b/k6-tests/ncmp/6-id-search-public-property.js
@@ -19,6 +19,7 @@
  */
 
 import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
     vus: 5,
@@ -29,14 +30,20 @@ export const options = {
     },
 };
 
-export default function() {
+export default function () {
     const search_filter = {
         "cmHandleQueryParameters": [
             {
                 "conditionName": "hasAllProperties",
-                "conditionParameters": [ {"Color": "yellow"}, {"Size": "small"} ]
+                "conditionParameters": [{"Color": "yellow"}, {"Size": "small"}]
             }
         ]
     };
     searchRequest('id-searches', JSON.stringify(search_filter));
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/7-search-public-property.js b/k6-tests/ncmp/7-search-public-property.js
index 3e052dabe0..9a8e339c6c 100644
--- a/k6-tests/ncmp/7-search-public-property.js
+++ b/k6-tests/ncmp/7-search-public-property.js
@@ -19,6 +19,7 @@
  */
 
 import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
     vus: 5,
@@ -29,14 +30,20 @@ export const options = {
     },
 };
 
-export default function() {
+export default function () {
     const search_filter = {
         "cmHandleQueryParameters": [
             {
                 "conditionName": "hasAllProperties",
-                "conditionParameters": [ {"Color": "yellow"}, {"Size": "small"} ]
+                "conditionParameters": [{"Color": "yellow"}, {"Size": "small"}]
             }
         ]
     };
     searchRequest('searches', JSON.stringify(search_filter));
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/8-id-search-module.js b/k6-tests/ncmp/8-id-search-module.js
index e99fbfeb28..299fe62b89 100644
--- a/k6-tests/ncmp/8-id-search-module.js
+++ b/k6-tests/ncmp/8-id-search-module.js
@@ -19,6 +19,7 @@
  */
 
 import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
     vus: 5,
@@ -29,14 +30,20 @@ export const options = {
     },
 };
 
-export default function() {
+export default function () {
     const search_filter = {
         "cmHandleQueryParameters": [
             {
                 "conditionName": "hasAllModules",
-                "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+                "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
             }
         ]
     };
     searchRequest('id-searches', JSON.stringify(search_filter));
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/9-search-module.js b/k6-tests/ncmp/9-search-module.js
index 03eb9a67ab..d340bf2c7a 100644
--- a/k6-tests/ncmp/9-search-module.js
+++ b/k6-tests/ncmp/9-search-module.js
@@ -19,6 +19,7 @@
  */
 
 import { searchRequest } from './search-base.js';
+import { makeCustomSummaryReport } from "./utils.js";
 
 export const options = {
     vus: 5,
@@ -29,14 +30,20 @@ export const options = {
     },
 };
 
-export default function() {
+export default function () {
     const search_filter = {
         "cmHandleQueryParameters": [
             {
                 "conditionName": "hasAllModules",
-                "conditionParameters": [ {"moduleName": "ietf-yang-types-1"} ]
+                "conditionParameters": [{"moduleName": "ietf-yang-types-1"}]
             }
         ]
     };
     searchRequest('searches', JSON.stringify(search_filter));
 }
+
+export function handleSummary(data) {
+    return {
+        stdout: makeCustomSummaryReport(data, options),
+    };
+}
\ No newline at end of file
diff --git a/k6-tests/ncmp/run-all-tests.sh b/k6-tests/ncmp/run-all-tests.sh
index f75cf2f701..80231b9809 100755
--- a/k6-tests/ncmp/run-all-tests.sh
+++ b/k6-tests/ncmp/run-all-tests.sh
@@ -31,13 +31,19 @@ ALL_TEST_SCRIPTS=( \
 
 pushd "$(dirname "$0")" || exit 1
 
+printf "Test Case\tCondition\tLimit\tActual\tResult\n" > summary.log
+
 number_of_failures=0
 for test_script in "${ALL_TEST_SCRIPTS[@]}"; do
   echo "k6 run $test_script"
-  k6 --quiet run "$test_script" || ((number_of_failures++))
-  echo
+  k6 --quiet run -e K6_MODULE_NAME="$test_script" "$test_script" >> summary.log || ((number_of_failures++))
 done
 
+echo '##############################################################################################################################'
+echo '##                             K 6   P E R F O R M A N C E   T E S T   R E S U L T S                                        ##'
+echo '##############################################################################################################################'
+awk -F$'\t' '{printf "%-40s%-50s%-20s%-10s%-6s\n", $1, $2, $3, $4, $5}' summary.log
+
 popd || exit 1
 
 echo "NCMP TEST FAILURES: $number_of_failures"
diff --git a/k6-tests/ncmp/search-base.js b/k6-tests/ncmp/search-base.js
index 9c1b247a9d..19a6a5aa6f 100644
--- a/k6-tests/ncmp/search-base.js
+++ b/k6-tests/ncmp/search-base.js
@@ -24,7 +24,7 @@ import { NCMP_BASE_URL, TOTAL_CM_HANDLES } from './utils.js';
 
 export function searchRequest(searchType, searchFilter) {
     const response = http.post(NCMP_BASE_URL + '/ncmp/v1/ch/' + searchType, searchFilter, {
-        headers: { 'Content-Type': 'application/json' },
+        headers: {'Content-Type': 'application/json'},
     });
     check(response, {
         'status equals 200': (r) => r.status === 200,
diff --git a/k6-tests/ncmp/utils.js b/k6-tests/ncmp/utils.js
index 3b61b217e6..18a8940523 100644
--- a/k6-tests/ncmp/utils.js
+++ b/k6-tests/ncmp/utils.js
@@ -39,3 +39,23 @@ export function makeBatchOfCmHandleIds(batchSize, batchNumber) {
 export function getRandomCmHandleId() {
     return 'ch-' + (Math.floor(Math.random() * TOTAL_CM_HANDLES) + 1);
 }
+
+function removeBracketsAndQuotes(str) {
+    return str.replace(/\[|\]|"/g, '');
+}
+
+export function makeCustomSummaryReport(data, options) {
+    const moduleName = `${__ENV.K6_MODULE_NAME}`;
+    let body = ``;
+    for (const condition in options.thresholds) {
+        let limit = JSON.stringify(options.thresholds[condition])
+        limit = removeBracketsAndQuotes(limit)
+        let limitKey = limit.split(' ')[0]
+        const actual = Math.ceil(data.metrics[condition].values[limitKey])
+        const result = data.metrics[condition].thresholds[limit].ok ? 'PASS' : 'FAIL'
+        const row = `${moduleName}\t${condition}\t${limit}\t${actual}\t${result}\n`;
+        body += row;
+    }
+    return body;
+}
+
diff --git a/k6-tests/teardown.sh b/k6-tests/teardown.sh
index 87cca72529..45422f9d12 100755
--- a/k6-tests/teardown.sh
+++ b/k6-tests/teardown.sh
@@ -18,9 +18,6 @@
 echo '================================== docker info =========================='
 docker ps -a
 
-echo '================================== CPS-NCMP Logs ========================'
-docker logs cps-and-ncmp
-
 echo 'Stopping, Removing all running containers...'
 docker stop $(docker ps -aq) && docker rm $(docker ps -aq)