handle results format

Issue-ID: VNFSDK-195
Change-Id: I31f07515b252247fe777791d883d37ba9ba1bd6a
Signed-off-by: Moshe <moshehoa@amdocs.com>

7 files changed, 36 insertions, 19 deletions

diff --git a/dist/vnftest-0.1.dev0-py2.7.egg b/dist/vnftest-0.1.dev0-py2.7.egg
index f237cd8..ab2a2dc 100644
--- a/dist/vnftest-0.1.dev0-py2.7.egg
+++ b/dist/vnftest-0.1.dev0-py2.7.egg
diff --git a/vnftest.egg-info/SOURCES.txt b/vnftest.egg-info/SOURCES.txt
index 04d6f8a..4bb5c16 100644
--- a/vnftest.egg-info/SOURCES.txt
+++ b/vnftest.egg-info/SOURCES.txt
@@ -66,5 +66,4 @@ vnftest/onap/steps/base.py
 vnftest/onap/steps/onap_api_call.py
 vnftest/onap/steps/dummy/__init__.py
 vnftest/onap/steps/dummy/dummy.py
-vnftest/onap/steps/onboard/__init__.py
-vnftest/tests/__init__.py
\ No newline at end of file
+vnftest/onap/steps/onboard/__init__.py
\ No newline at end of file
diff --git a/vnftest/cmd/commands/task.py b/vnftest/cmd/commands/task.py
index cc77bb8..c0aa63d 100644
--- a/vnftest/cmd/commands/task.py
+++ b/vnftest/cmd/commands/task.py
@@ -63,7 +63,6 @@ class TaskCommands(object):     # pragma: no cover
             LOG.info('Task SUCCESS')
         else:
             LOG.info('Task FAILED')
-            raise RuntimeError('Task Failed')
 
     def _write_error_data(self, error):
         data = {'status': 2, 'result': str(error)}
diff --git a/vnftest/common/exceptions.py b/vnftest/common/exceptions.py
index 6273cd3..9dc1ead 100644
--- a/vnftest/common/exceptions.py
+++ b/vnftest/common/exceptions.py
@@ -59,3 +59,7 @@ class VnftestException(Exception):
 class FunctionNotImplemented(VnftestException):
     message = ('The function "%(function_name)s" is not implemented in '
                '"%(class_name)" class.')
+
+
+class MandatoryKeyException(VnftestException):
+    message = 'No value found for key %(key_name)" in "%(dict_str)"'
diff --git a/vnftest/onap/core/task.py b/vnftest/onap/core/task.py
index 2d3033a..8e34665 100644
--- a/vnftest/onap/core/task.py
+++ b/vnftest/onap/core/task.py
@@ -37,6 +37,7 @@ from vnftest.onap.contexts.base import Context
 from vnftest.onap.contexts.csar import CSARContext
 from vnftest.onap.runners import base as base_runner
 from vnftest.onap.runners.duration import DurationRunner
+from vnftest.onap.runners.iteration import IterationRunner
 from vnftest.common.constants import CONF_FILE
 from vnftest.common.yaml_loader import yaml_load
 from vnftest.dispatcher.base import Base as DispatcherBase
@@ -136,8 +137,8 @@ class Task(object):     # pragma: no cover
                 LOG.error('Testcase: "%s" FAILED!!!', case_name, exc_info=True)
                 testcases[case_name] = {'criteria': 'FAIL', 'tc_data': []}
             else:
-                LOG.info('Testcase: "%s" SUCCESS!!!', case_name)
-                testcases[case_name] = {'criteria': 'PASS', 'tc_data': data}
+                criteria = self.evaluate_task_criteria(data)
+                testcases[case_name] = {'criteria': criteria, 'tc_data': data}
 
             if args.keep_deploy:
                 # keep deployment, forget about stack
@@ -238,6 +239,13 @@ class Task(object):     # pragma: no cover
         else:
             return 'PASS'
 
+    def evaluate_task_criteria(self, steps_result_list):
+        for step_result in steps_result_list:
+            errors_list = step_result['errors']
+            if errors_list is not None and len(errors_list) > 0:
+                return 'FAIL'
+        return 'PASS'
+
     def _do_output(self, output_config, result):
         dispatchers = DispatcherBase.get(output_config)
 
@@ -320,9 +328,9 @@ class Task(object):     # pragma: no cover
 
     def run_one_step(self, step_cfg, output_file):
         """run one step using context"""
-        # default runner is Duration
+        # default runner is Iteration
         if 'runner' not in step_cfg:
-            step_cfg['runner'] = dict(type="Duration", duration=1000000000)
+            step_cfg['runner'] = dict(type="Iteration", iterations=1)
         runner_cfg = step_cfg['runner']
         runner_cfg['output_filename'] = output_file
         options = step_cfg.get('options', {})
diff --git a/vnftest/onap/runners/iteration.py b/vnftest/onap/runners/iteration.py
index 9bac92e..c0bd74f 100644
--- a/vnftest/onap/runners/iteration.py
+++ b/vnftest/onap/runners/iteration.py
@@ -67,17 +67,21 @@ def _worker_process(queue, cls, method_name, step_cfg,
                        "sequence": sequence})
 
             data = {}
-            errors = ""
+            errors = []
 
             try:
                 result = method(data)
+                if result:
+                    # add timeout for put so we don't block test
+                    # if we do timeout we don't care about dropping individual KPIs
+                    output_queue.put(result, True, QUEUE_PUT_TIMEOUT)
             except AssertionError as assertion:
                 # SLA validation failed in step, determine what to do now
                 if sla_action == "assert":
                     raise
                 elif sla_action == "monitor":
                     LOG.warning("SLA validation failed: %s", assertion.args)
-                    errors = assertion.args
+                    errors.append(assertion.args)
                 elif sla_action == "rate-control":
                     try:
                         step_cfg['options']['rate']
@@ -89,13 +93,8 @@ def _worker_process(queue, cls, method_name, step_cfg,
                     sequence = 1
                     continue
             except Exception:
-                errors = traceback.format_exc()
+                errors.append(traceback.format_exc())
                 LOG.exception("")
-            else:
-                if result:
-                    # add timeout for put so we don't block test
-                    # if we do timeout we don't care about dropping individual KPIs
-                    output_queue.put(result, True, QUEUE_PUT_TIMEOUT)
 
             time.sleep(interval)
 
diff --git a/vnftest/onap/steps/onap_api_call.py b/vnftest/onap/steps/onap_api_call.py
index ecb2ce3..ec53455 100644
--- a/vnftest/onap/steps/onap_api_call.py
+++ b/vnftest/onap/steps/onap_api_call.py
@@ -18,6 +18,8 @@ import time
 import os
 import yaml
 import copy
+
+from vnftest.common.exceptions import MandatoryKeyException
 from vnftest.onap.steps import base
 from vnftest.common import rest_client
 from vnftest.common import constants as consts
@@ -45,16 +47,17 @@ class OnapApiCall(base.Step):
         self.output = options.get("output")
         self.setup_done = True
 
-    def run(self, args):
+    def run(self, result):
         if not self.setup_done:
             self.setup()
+        result['output'] = {}
         params = copy.deepcopy(consts.component_constants)
         for input_parameter in self.input:
             param_name = input_parameter['parameter_name']
             param_value = input_parameter['value']
             params[param_name] = param_value
-        result = self.execute_operation(params)
-        result_body = result['body']
+        execution_result = self.execute_operation(params)
+        result_body = execution_result['body']
         for output_parameter in self.output:
             param_name = output_parameter['parameter_name']
             param_path = output_parameter['path']
@@ -62,7 +65,12 @@ class OnapApiCall(base.Step):
             param_value = result_body
             for path_element in path_list:
                 param_value = param_value[path_element]
-        self.context_cfg[param_name] = param_value
+            if param_value is None:
+                raise MandatoryKeyException(
+                    key_name='param_path', class_name=str(result_body))
+            self.context_cfg[param_name] = param_value
+            result['output'][param_name] = param_value
+        result['status'] = 'PASS'
 
     def execute_operation(self, params, attempt=0):
         try: