Error reason added to the test reports

Error reason added to the test reports

Issue-ID: TEST-402
Signed-off-by: Lukasz Rajewski <lukasz.rajewski@t-mobile.pl>
Change-Id: I024626764b134d9fe7607988cf46918414f1deb3

5 files changed, 131 insertions, 44 deletions

diff --git a/src/onaptests/scenario/scenario_base.py b/src/onaptests/scenario/scenario_base.py
index ca6d898..f1b4455 100644
--- a/src/onaptests/scenario/scenario_base.py
+++ b/src/onaptests/scenario/scenario_base.py
@@ -47,14 +47,18 @@ class ScenarioBase(testcase.TestCase):
                     test_phase()
                     self.result += 50
                 except OnapTestException as exc:
-                    self.__logger.exception("%s on %s", exc.error_message, phase_name)
-                except SDKException:
-                    self.__logger.exception("SDK Exception on %s", phase_name)
-                except Exception as e:
-                    self.__logger.exception("General Exception on %s", phase_name)
+                    self.__logger.exception("Test Exception %s on %s", str(exc), phase_name)
+                    self.__logger.info("ROOT CAUSE")
+                    self.__logger.info(exc.root_cause)
+                except SDKException as exc:
+                    self.__logger.exception("SDK Exception %s on %s", str(exc), phase_name)
+                    self.__logger.info("ROOT CAUSE")
+                    self.__logger.info(str(exc))
+                except Exception as exc:
+                    self.__logger.exception("General Exception %s on %s", str(exc), phase_name)
                     if self.general_exception:
-                        e = ExceptionGroup("General Exceptions", [self.general_exception, e])  # noqa
-                    self.general_exception = e
+                        exc = ExceptionGroup("General Exceptions", [self.general_exception, exc])  # noqa
+                    self.general_exception = exc
         finally:
             self.stop_time = time.time()
             self.__logger.info(f"{self.scenario_name} Execution {self.result}% Completed")
diff --git a/src/onaptests/steps/base.py b/src/onaptests/steps/base.py
index 00229ba..c615047 100644
--- a/src/onaptests/steps/base.py
+++ b/src/onaptests/steps/base.py
@@ -40,6 +40,7 @@ class StoreStateHandler(ABC):
             else:
                 self._state_execute = True
             initial_exception = None
+            error_reason = []
             try:
                 execution_status: Optional[ReportStepStatus] = ReportStepStatus.FAIL
                 if cleanup:
@@ -81,11 +82,16 @@ class StoreStateHandler(ABC):
                 if initial_exception:
                     substep_exc = OnapTestExceptionGroup("Cleanup Exceptions",
                                                          [initial_exception, substep_exc])
+                error_reason = substep_exc.root_cause
                 raise substep_exc
             except (OnapTestException, SDKException) as test_exc:
                 if initial_exception:
                     test_exc = OnapTestExceptionGroup("Cleanup Exceptions",
                                                       [initial_exception, test_exc])
+                if isinstance(test_exc, OnapTestException):
+                    error_reason = test_exc.root_cause
+                else:
+                    error_reason = [str(test_exc)]
                 raise test_exc
             finally:
                 if not cleanup:
@@ -95,7 +101,8 @@ class StoreStateHandler(ABC):
                         step_description=self._step_title(cleanup),
                         step_execution_status=execution_status,
                         step_execution_duration=time.time() - self._start_cleanup_time,
-                        step_component=self.component
+                        step_component=self.component,
+                        step_error_reason=error_reason
                     )
                 else:
                     if not self._start_execution_time:
@@ -110,7 +117,8 @@ class StoreStateHandler(ABC):
                         step_execution_status=(execution_status if execution_status else
                                                ReportStepStatus.FAIL),
                         step_execution_duration=time.time() - self._start_execution_time,
-                        step_component=self.component
+                        step_component=self.component,
+                        step_error_reason=error_reason
                     )
             wrapper._is_wrapped = True
         return wrapper
@@ -349,18 +357,19 @@ class BaseStep(StoreStateHandler, ABC):
         Override this method and remember to call `super().execute()` before.
 
         """
-        substep_error = False
+        substep_exceptions = []
         for step in self._steps:
             try:
                 step.execute()
             except (OnapTestException, SDKException) as substep_err:
-                substep_error = True
                 if step._break_on_error:
-                    raise SubstepExecutionException from substep_err
-                self._logger.exception(substep_err)
+                    raise SubstepExecutionException("", substep_err) # noqa: W0707
+                substep_exceptions.append(substep_err)
         if self._steps:
-            if substep_error and self._break_on_error:
-                raise SubstepExecutionException("Cannot continue due to failed substeps")
+            if len(substep_exceptions) > 0 and self._break_on_error:
+                if len(substep_exceptions) == 1:
+                    raise SubstepExecutionException("", substep_exceptions[0])
+                raise SubstepExecutionExceptionGroup("", substep_exceptions)
             self._log_execution_state("CONTINUE")
         self._substeps_executed = True
         self._start_execution_time = time.time()
@@ -381,13 +390,13 @@ class BaseStep(StoreStateHandler, ABC):
                     step._default_cleanup_handler()
             except (OnapTestException, SDKException) as substep_err:
                 try:
-                    raise SubstepExecutionException from substep_err
+                    raise SubstepExecutionException("", substep_err) # noqa: W0707
                 except Exception as e:
                     exceptions_to_raise.append(e)
         if len(exceptions_to_raise) > 0:
             if len(exceptions_to_raise) == 1:
                 raise exceptions_to_raise[0]
-            raise SubstepExecutionExceptionGroup("Substep Exceptions", exceptions_to_raise)
+            raise SubstepExecutionExceptionGroup("", exceptions_to_raise)
 
     def execute(self) -> None:
         """Step's execute.
diff --git a/src/onaptests/steps/reports_collection.py b/src/onaptests/steps/reports_collection.py
index 2f92660..4f0965f 100644
--- a/src/onaptests/steps/reports_collection.py
+++ b/src/onaptests/steps/reports_collection.py
@@ -25,6 +25,7 @@ class Report:
     step_execution_status: ReportStepStatus
     step_execution_duration: float
     step_component: str
+    step_error_reason: List[str]
 
 
 class ReportsCollection:
@@ -103,7 +104,8 @@ class ReportsCollection:
                     'description': step_report.step_description,
                     'status': step_report.step_execution_status.value,
                     'duration': step_report.step_execution_duration,
-                    'component': step_report.step_component
+                    'component': step_report.step_component,
+                    'reason': step_report.step_error_reason
                 }
                 for step_report in reversed(self.report)
             ]
diff --git a/src/onaptests/templates/reporting/reporting.html.j2 b/src/onaptests/templates/reporting/reporting.html.j2
index 246f362..dab7b1b 100644
--- a/src/onaptests/templates/reporting/reporting.html.j2
+++ b/src/onaptests/templates/reporting/reporting.html.j2
@@ -31,6 +31,18 @@
             <tr {% if  step_report.step_execution_status.value == 'FAIL' %} class="has-background-danger" {% elif step_report.step_execution_status.value == 'PASS' %} class="has-background-success-light" {% else %} class="has-background-warning-light" {% endif %}>
             <td>
             {{ step_report.step_description }}
+            {% if  step_report.step_execution_status.value == 'FAIL' and (step_report.step_error_reason | length) > 0 %}
+            <table class="table is-fullwidth is-striped is-hoverable">
+            {% for error_reason in step_report.step_error_reason %}
+            <tr class="has-background-danger">
+            <td></td>
+            </tr>
+            <tr class="has-background-danger-light">
+            <td>{{ error_reason }}</td>
+            </tr>
+            {% endfor %}
+            </table>
+            {% endif %}
             </td>
             <td>
             {{ step_report.step_execution_status.value }}
diff --git a/src/onaptests/utils/exceptions.py b/src/onaptests/utils/exceptions.py
index aeae9d1..0d7ecee 100644
--- a/src/onaptests/utils/exceptions.py
+++ b/src/onaptests/utils/exceptions.py
@@ -11,106 +11,166 @@ __author__ = "Morgan Richomme <morgan.richomme@orange.com>"
 
 class OnapTestException(Exception):
     """Parent Class for all Onap Test Exceptions."""
-    error_message = 'Generic OnapTest exception'
-
-
-class OnapTestExceptionGroup(ExceptionGroup, OnapTestException):  # noqa
+    def __init__(self, __message='Generic OnapTest exception', *args, **kwargs): # noqa: W1113
+        super().__init__(__message, *args, **kwargs)
+        self.error_message = __message
+        if self.error_message:
+            self.error_message = str(self.error_message)
+
+    def __str__(self):
+        values = self.root_cause
+        if len(values) == 0:
+            return ""
+        if len(values) == 1:
+            return str(values[0])
+        return str(values)
+
+    @property
+    def root_cause(self):
+        """Real reason of the test exception"""
+        return [self.error_message]
+
+
+class OnapTestExceptionGroup(OnapTestException, ExceptionGroup):  # noqa
     """Group of Onap Test Exceptions."""
-    error_message = 'Generic OnapTest exception group'
-
-
-class SkipExecutionException(OnapTestException):
-    """Used only for validation purposes"""
+    def __init__(self, __message='Generic OnapTest exception group', __exceptions=None):
+        super().__init__(__message, __exceptions)
 
 
 class TestConfigurationException(OnapTestException):
     """Raise when configuration of the use case is incomplete or buggy."""
-    error_message = 'Configuration error'
+    def __init__(self, __message='Configuration error'):
+        super().__init__(__message)
 
 
 class ServiceDistributionException(OnapTestException):
     """Service not properly distributed."""
-    error_message = 'Service not well distributed'
+    def __init__(self, __message='Service not well distributed'):
+        super().__init__(__message)
 
 
 class ServiceInstantiateException(OnapTestException):
     """Service cannot be instantiated."""
-    error_message = 'Service instantiation error'
+    def __init__(self, __message='Service instantiation error'):
+        super().__init__(__message)
 
 
 class ServiceCleanupException(OnapTestException):
     """Service cannot be cleaned."""
-    error_message = 'Service not well cleaned up'
+    def __init__(self, __message='Service not well cleaned up'):
+        super().__init__(__message)
 
 
 class VnfInstantiateException(OnapTestException):
     """VNF cannot be instantiated."""
-    error_message = 'VNF instantiation error'
+    def __init__(self, __message='VNF instantiation error'):
+        super().__init__(__message)
 
 
 class VnfCleanupException(OnapTestException):
     """VNF cannot be cleaned."""
-    error_message = "VNF can't be cleaned"
+    def __init__(self, __message="VNF can't be cleaned"):
+        super().__init__(__message)
 
 
 class VfModuleInstantiateException(OnapTestException):
     """VF Module cannot be instantiated."""
-    error_message = 'VF Module instantiation error'
+    def __init__(self, __message='VF Module instantiation error'):
+        super().__init__(__message)
 
 
 class VfModuleCleanupException(OnapTestException):
     """VF Module cannot be cleaned."""
-    error_message = "VF Module can't be cleaned"
+    def __init__(self, __message="VF Module can't be cleaned"):
+        super().__init__(__message)
 
 
 class NetworkInstantiateException(OnapTestException):
     """Network cannot be instantiated."""
-    error_message = 'Network instantiation error'
+    def __init__(self, __message='Network instantiation error'):
+        super().__init__(__message)
 
 
 class NetworkCleanupException(OnapTestException):
     """Network cannot be cleaned."""
-    error_message = "Network can't be cleaned"
+    def __init__(self, __message="Network can't be cleaned"):
+        super().__init__(__message)
 
 
 class ProfileInformationException(OnapTestException):
     """Missing k8s profile information."""
-    error_message = 'Missing k8s profile information'
+    def __init__(self, __message='Missing k8s profile information'):
+        super().__init__(__message)
 
 
 class ProfileCleanupException(OnapTestException):
     """K8s profile cannot be cleaned."""
-    error_message = "Profile can't be cleaned"
+    def __init__(self, __message="Profile can't be cleaned"):
+        super().__init__(__message)
 
 
 class EnvironmentPreparationException(OnapTestException):
     """Test environment preparation exception."""
-    error_message = "Test can't be run properly due to preparation error"
+    def __init__(self, __message="Test can't be run properly due to preparation error"):
+        super().__init__(__message)
 
 
 class SubstepExecutionException(OnapTestException):
     """Exception raised if substep execution fails."""
+    def __init__(self, __message, __exception):
+        super().__init__(__message)
+        self.sub_exception = __exception
+
+    @property
+    def root_cause(self):
+        """Real reason of the test exception"""
+        if hasattr(self, "sub_exception"):
+            if isinstance(self.sub_exception, OnapTestException):
+                return self.sub_exception.root_cause
+            return [str(self.sub_exception)]
+        return super().root_cause
 
 
 class SubstepExecutionExceptionGroup(ExceptionGroup, SubstepExecutionException):  # noqa
     """Group of Substep Exceptions."""
+    def __init__(self, __message="Substeps group has failed",
+                 __exceptions=None):
+        super().__init__(__message, __exceptions)
+        self.sub_exceptions = __exceptions
+
+    @property
+    def root_cause(self):
+        """Real reason of the test exception"""
+        if self.sub_exceptions:
+            values = []
+            for exc in self.sub_exceptions:
+                if isinstance(exc, OnapTestException):
+                    values.extend(exc.root_cause)
+                else:
+                    values.append(str(exc))
+            return values
+        return super().root_cause
 
 
 class EnvironmentCleanupException(OnapTestException):
     """Test environment cleanup exception."""
-    error_message = "Test couldn't finish a cleanup"
+    def __init__(self, __message="Test couldn't finish a cleanup"):
+        super().__init__(__message)
 
 
 class PolicyException(OnapTestException):
     """Policy exception."""
-    error_message = "Problem with policy module"
+    def __init__(self, __message="Problem with policy module"):
+        super().__init__(__message)
 
 
 class DcaeException(OnapTestException):
     """DCAE exception."""
-    error_message = "Problem with DCAE module"
+    def __init__(self, __message="Problem with DCAE module"):
+        super().__init__(__message)
 
 
 class StatusCheckException(OnapTestException):
     """Status Check exception."""
-    error_message = "Namespace status check has failed"
+    def __init__(self, __message="Namespace status check has failed"):
+        super().__init__(__message)