Timeout on orchestration requests

Use wait_for_finish method to raise an exception if orchestration requests took more than 10 minutes

Issue-ID: TEST-316
Signed-off-by: Michal Jagiello <michal.jagiello@t-mobile.pl>
Change-Id: I0d82d91b0f7104caf32e5905d5950047d7551a7b

6 files changed, 75 insertions, 47 deletions

diff --git a/src/onaptests/configuration/settings.py b/src/onaptests/configuration/settings.py
index 609ca66..5cea0a6 100644
--- a/src/onaptests/configuration/settings.py
+++ b/src/onaptests/configuration/settings.py
@@ -45,3 +45,5 @@ TILLER_HOST = "localhost"
 K8S_CONFIG = None  # None means it will use default config (~/.kube/config)
 K8S_NAMESPACE = "onap"  # Kubernetes namespace
 #SOCK_HTTP = "socks5h://127.0.0.1:8091"
+
+ORCHESTRATION_REQUEST_TIMEOUT = 60.0 * 10  # Ten minutes in seconds
diff --git a/src/onaptests/steps/instantiate/service_ala_carte.py b/src/onaptests/steps/instantiate/service_ala_carte.py
index eb3c882..858db2d 100644
--- a/src/onaptests/steps/instantiate/service_ala_carte.py
+++ b/src/onaptests/steps/instantiate/service_ala_carte.py
@@ -81,8 +81,14 @@ class ServiceAlaCarteInstantiateStep(BaseStep):
             vid_project,
             service_instance_name=settings.SERVICE_INSTANCE_NAME
         )
-        while not service_instantiation.finished:
-            time.sleep(10)
+        try:
+            service_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+        except TimeoutError:
+            self._logger.error("Service instantiation %s timed out", self.service_instance_name)
+            raise onap_test_exceptions.ServiceInstantiateException
+        if service_instantiation.failed:
+            self._logger.error("Service instantiation %s failed", self.service_instance_name)
+            raise onap_test_exceptions.ServiceInstantiateException
 
 
 class YamlTemplateServiceAlaCarteInstantiateStep(YamlTemplateBaseStep):
@@ -225,9 +231,13 @@ class YamlTemplateServiceAlaCarteInstantiateStep(YamlTemplateBaseStep):
             vid_project,
             service_instance_name=self.service_instance_name
         )
-        while not service_instantiation.finished:
-            time.sleep(10)
+        try:
+            service_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+        except TimeoutError:
+            self._logger.error("Service instantiation %s timed out", self.service_instance_name)
+            raise onap_test_exceptions.ServiceCleanupException
         if service_instantiation.failed:
+            self._logger.error("Service instantiation %s failed", self.service_instance_name)
             raise onap_test_exceptions.ServiceInstantiateException
         else:
             service_subscription: ServiceSubscription = customer.get_service_subscription_by_service_type(self.service_name)
@@ -242,12 +252,11 @@ class YamlTemplateServiceAlaCarteInstantiateStep(YamlTemplateBaseStep):
 
         """
         service_deletion = self._service_instance.delete()
-        nb_try = 0
-        nb_try_max = 30
-        while not service_deletion.finished and nb_try < nb_try_max:
-            self._logger.info("Wait for Service deletion")
-            nb_try += 1
-            time.sleep(15)
+        try:
+            service_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+        except TimeoutError:
+            self._logger.error("Service deletion %s timed out", self._service_instance_name)
+            raise onap_test_exceptions.ServiceCleanupException
         if service_deletion.finished:
             self._logger.info("Service %s deleted", self._service_instance_name)
         else:
diff --git a/src/onaptests/steps/instantiate/service_macro.py b/src/onaptests/steps/instantiate/service_macro.py
index 526eecc..14470f8 100644
--- a/src/onaptests/steps/instantiate/service_macro.py
+++ b/src/onaptests/steps/instantiate/service_macro.py
@@ -179,8 +179,13 @@ class YamlTemplateServiceMacroInstantiateStep(YamlTemplateBaseStep):
             tenant=tenant,
             service_instance_name=self.service_instance_name
         )
-        service_instantiation.wait_for_finish(timeout=settings.INSTANTIATION_TIMEOUT)
+        try:
+            service_instantiation.wait_for_finish(timeout=settings.ORCHESTRATION_REQUEST_TIMEOUT)
+        except TimeoutError:
+            self._logger.error("Service instantiation %s timed out", self.service_instance_name)
+            raise onap_test_exceptions.ServiceInstantiateException
         if service_instantiation.failed:
+            self._logger.error("Service instantiation %s failed", self.service_instance_name)
             raise onap_test_exceptions.ServiceInstantiateException
         else:
             service_subscription: ServiceSubscription = customer.get_service_subscription_by_service_type(self.service_name)
@@ -195,7 +200,11 @@ class YamlTemplateServiceMacroInstantiateStep(YamlTemplateBaseStep):
 
         """
         service_deletion = self._service_instance.delete()
-        service_deletion.wait_for_finish(timeout=600)
+        try:
+            service_deletion.wait_for_finish(timeout=settings.ORCHESTRATION_REQUEST_TIMEOUT)
+        except TimeoutError:
+            self._logger.error("Service deletion %s timed out", self._service_instance_name)
+            raise onap_test_exceptions.ServiceCleanupException
         if service_deletion.finished:
             self._logger.info("Service %s deleted", self._service_instance_name)
         else:
diff --git a/src/onaptests/steps/instantiate/vf_module_ala_carte.py b/src/onaptests/steps/instantiate/vf_module_ala_carte.py
index 2c1834e..ee20bc6 100644
--- a/src/onaptests/steps/instantiate/vf_module_ala_carte.py
+++ b/src/onaptests/steps/instantiate/vf_module_ala_carte.py
@@ -1,4 +1,3 @@
-import time
 from typing import Iterable
 from uuid import uuid4
 from yaml import load
@@ -143,10 +142,14 @@ class YamlTemplateVfModuleAlaCarteInstantiateStep(YamlTemplateBaseStep):
                     tenant,
                     self._service_instance_name,
                     vnf_parameters=self.get_vnf_parameters(vnf_instance.vnf.name))
-            while not vf_module_instantiation.finished:
-                time.sleep(10)
-            if vf_module_instantiation.failed:
-                raise onap_test_exceptions.VfModuleInstantiateException
+                try:
+                    vf_module_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+                    if vf_module_instantiation.failed:
+                        self._logger.error("VfModule instantiation %s failed", vf_module.name)
+                        raise onap_test_exceptions.VfModuleInstantiateException
+                except TimeoutError:
+                    self._logger.error("VfModule instantiation %s timed out", vf_module.name)
+                    raise onap_test_exceptions.VfModuleInstantiateException
 
     @YamlTemplateBaseStep.store_state(cleanup=True)
     def cleanup(self) -> None:
@@ -164,16 +167,14 @@ class YamlTemplateVfModuleAlaCarteInstantiateStep(YamlTemplateBaseStep):
                 self._logger.info("Delete VF Module %s",
                                   vf_module.name)
                 vf_module_deletion = vf_module.delete()
-                nb_try = 0
-                nb_try_max = 30
-
-                while not vf_module_deletion.finished and nb_try < nb_try_max:
-                    self._logger.info("Wait for vf module deletion")
-                    nb_try += 1
-                    time.sleep(20)
-                if vf_module_deletion.finished:
+
+                try:
+                    vf_module_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+                    if vf_module_deletion.failed:
+                        self._logger.error("VfModule deletion %s failed", vf_module.name)
+                        raise onap_test_exceptions.VfModuleCleanupException
                     self._logger.info("VfModule %s deleted", vf_module.name)
-                else:
-                    self._logger.error("VfModule deletion %s failed", vf_module.name)
+                except TimeoutError:
+                    self._logger.error("VfModule deletion %s timed out", vf_module.name)
                     raise onap_test_exceptions.VfModuleCleanupException
         super().cleanup()
diff --git a/src/onaptests/steps/instantiate/vl_ala_carte.py b/src/onaptests/steps/instantiate/vl_ala_carte.py
index f9ac560..fd040aa 100644
--- a/src/onaptests/steps/instantiate/vl_ala_carte.py
+++ b/src/onaptests/steps/instantiate/vl_ala_carte.py
@@ -1,4 +1,3 @@
-import time
 import re
 from typing import Iterable
 from uuid import uuid4
@@ -141,9 +140,13 @@ class YamlTemplateVlAlaCarteInstantiateStep(YamlTemplateBaseStep):
                 platform,
                 network_instance_name=f"{self.service_instance_name}_net_{idx}",
                 subnets=self.get_subnets(network.name))
-            while not net_instantiation.finished:
-                time.sleep(10)
-            if net_instantiation.failed:
+            try:
+                net_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+                if net_instantiation.failed:
+                    self._logger.error("VL instantiation %s failed", net_instantiation.name)
+                    raise onap_test_exceptions.NetworkInstantiateException
+            except TimeoutError:
+                self._logger.error("VL instantiation %s timed out", net_instantiation.name)
                 raise onap_test_exceptions.NetworkInstantiateException
 
     @YamlTemplateBaseStep.store_state(cleanup=True)
@@ -158,7 +161,11 @@ class YamlTemplateVlAlaCarteInstantiateStep(YamlTemplateBaseStep):
                 self._logger.info("Start network deletion %s",net_instance.name)
                 net_deletion = net_instance.delete()
                 try:
-                    net_deletion.wait_for_finish()
+                    net_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+                    if net_deletion.failed:
+                        self._logger.error("VL deletion %s failed", net_instance.name)
+                        raise onap_test_exceptions.NetworkCleanupException
                 except TimeoutError:
+                    self._logger.error("VL deletion %s timed out", net_instance.name)
                     raise onap_test_exceptions.NetworkCleanupException
             super().cleanup()
diff --git a/src/onaptests/steps/instantiate/vnf_ala_carte.py b/src/onaptests/steps/instantiate/vnf_ala_carte.py
index 64ea090..9dc062a 100644
--- a/src/onaptests/steps/instantiate/vnf_ala_carte.py
+++ b/src/onaptests/steps/instantiate/vnf_ala_carte.py
@@ -1,4 +1,3 @@
-import time
 from uuid import uuid4
 from yaml import load
 
@@ -118,9 +117,13 @@ class YamlTemplateVnfAlaCarteInstantiateStep(YamlTemplateBaseStep):
                 cloud_region,
                 tenant,
                 f"{self.service_instance_name}_vnf_{idx}")
-            while not vnf_instantiation.finished:
-                time.sleep(10)
-            if vnf_instantiation.failed:
+            try:
+                vnf_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+                if vnf_instantiation.failed:
+                    self._logger.error("VNF instantiation %s failed", vnf.name)
+                    raise onap_test_exceptions.VnfInstantiateException
+            except TimeoutError:
+                self._logger.error("VNF instantiation %s timed out", vnf.name)
                 raise onap_test_exceptions.VnfInstantiateException
 
     @YamlTemplateBaseStep.store_state(cleanup=True)
@@ -133,16 +136,13 @@ class YamlTemplateVnfAlaCarteInstantiateStep(YamlTemplateBaseStep):
         """
         for vnf_instance in self._service_instance.vnf_instances:
             vnf_deletion = vnf_instance.delete()
-            nb_try = 0
-            nb_try_max = 30
-
-            while not vnf_deletion.finished and nb_try < nb_try_max:
-                self._logger.info("Wait for vnf deletion")
-                nb_try += 1
-                time.sleep(15)
-            if vnf_deletion.finished:
-                self._logger.info("VNF %s deleted", vnf_instance.name)
-            else:
-                self._logger.error("VNF deletion %s failed", vnf_instance.name)
+
+            try:
+                vnf_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT)
+                if vnf_deletion.failed:
+                    self._logger.error("VNF deletion %s failed", vnf_instance.name)
+                    raise onap_test_exceptions.VnfCleanupException
+            except TimeoutError:
+                self._logger.error("VNF deletion %s timed out", vnf_instance.name)
                 raise onap_test_exceptions.VnfCleanupException
         super().cleanup()