From 11eaf0bf2df74664ab9812d817e1d34e89112c0b Mon Sep 17 00:00:00 2001 From: Michal Jagiello Date: Fri, 19 Mar 2021 15:01:26 +0000 Subject: Timeout on orchestration requests Use wait_for_finish method to raise an exception if orchestration requests took more than 10 minutes Issue-ID: TEST-316 Signed-off-by: Michal Jagiello Change-Id: I0d82d91b0f7104caf32e5905d5950047d7551a7b --- src/onaptests/configuration/settings.py | 2 ++ .../steps/instantiate/service_ala_carte.py | 29 +++++++++++++------- src/onaptests/steps/instantiate/service_macro.py | 13 +++++++-- .../steps/instantiate/vf_module_ala_carte.py | 31 +++++++++++----------- src/onaptests/steps/instantiate/vl_ala_carte.py | 17 ++++++++---- src/onaptests/steps/instantiate/vnf_ala_carte.py | 30 ++++++++++----------- 6 files changed, 75 insertions(+), 47 deletions(-) diff --git a/src/onaptests/configuration/settings.py b/src/onaptests/configuration/settings.py index 609ca66..5cea0a6 100644 --- a/src/onaptests/configuration/settings.py +++ b/src/onaptests/configuration/settings.py @@ -45,3 +45,5 @@ TILLER_HOST = "localhost" K8S_CONFIG = None # None means it will use default config (~/.kube/config) K8S_NAMESPACE = "onap" # Kubernetes namespace #SOCK_HTTP = "socks5h://127.0.0.1:8091" + +ORCHESTRATION_REQUEST_TIMEOUT = 60.0 * 10 # Ten minutes in seconds diff --git a/src/onaptests/steps/instantiate/service_ala_carte.py b/src/onaptests/steps/instantiate/service_ala_carte.py index eb3c882..858db2d 100644 --- a/src/onaptests/steps/instantiate/service_ala_carte.py +++ b/src/onaptests/steps/instantiate/service_ala_carte.py @@ -81,8 +81,14 @@ class ServiceAlaCarteInstantiateStep(BaseStep): vid_project, service_instance_name=settings.SERVICE_INSTANCE_NAME ) - while not service_instantiation.finished: - time.sleep(10) + try: + service_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + except TimeoutError: + self._logger.error("Service instantiation %s timed out", self.service_instance_name) + raise onap_test_exceptions.ServiceInstantiateException + if service_instantiation.failed: + self._logger.error("Service instantiation %s failed", self.service_instance_name) + raise onap_test_exceptions.ServiceInstantiateException class YamlTemplateServiceAlaCarteInstantiateStep(YamlTemplateBaseStep): @@ -225,9 +231,13 @@ class YamlTemplateServiceAlaCarteInstantiateStep(YamlTemplateBaseStep): vid_project, service_instance_name=self.service_instance_name ) - while not service_instantiation.finished: - time.sleep(10) + try: + service_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + except TimeoutError: + self._logger.error("Service instantiation %s timed out", self.service_instance_name) + raise onap_test_exceptions.ServiceCleanupException if service_instantiation.failed: + self._logger.error("Service instantiation %s failed", self.service_instance_name) raise onap_test_exceptions.ServiceInstantiateException else: service_subscription: ServiceSubscription = customer.get_service_subscription_by_service_type(self.service_name) @@ -242,12 +252,11 @@ class YamlTemplateServiceAlaCarteInstantiateStep(YamlTemplateBaseStep): """ service_deletion = self._service_instance.delete() - nb_try = 0 - nb_try_max = 30 - while not service_deletion.finished and nb_try < nb_try_max: - self._logger.info("Wait for Service deletion") - nb_try += 1 - time.sleep(15) + try: + service_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + except TimeoutError: + self._logger.error("Service deletion %s timed out", self._service_instance_name) + raise onap_test_exceptions.ServiceCleanupException if service_deletion.finished: self._logger.info("Service %s deleted", self._service_instance_name) else: diff --git a/src/onaptests/steps/instantiate/service_macro.py b/src/onaptests/steps/instantiate/service_macro.py index 526eecc..14470f8 100644 --- a/src/onaptests/steps/instantiate/service_macro.py +++ b/src/onaptests/steps/instantiate/service_macro.py @@ -179,8 +179,13 @@ class YamlTemplateServiceMacroInstantiateStep(YamlTemplateBaseStep): tenant=tenant, service_instance_name=self.service_instance_name ) - service_instantiation.wait_for_finish(timeout=settings.INSTANTIATION_TIMEOUT) + try: + service_instantiation.wait_for_finish(timeout=settings.ORCHESTRATION_REQUEST_TIMEOUT) + except TimeoutError: + self._logger.error("Service instantiation %s timed out", self.service_instance_name) + raise onap_test_exceptions.ServiceInstantiateException if service_instantiation.failed: + self._logger.error("Service instantiation %s failed", self.service_instance_name) raise onap_test_exceptions.ServiceInstantiateException else: service_subscription: ServiceSubscription = customer.get_service_subscription_by_service_type(self.service_name) @@ -195,7 +200,11 @@ class YamlTemplateServiceMacroInstantiateStep(YamlTemplateBaseStep): """ service_deletion = self._service_instance.delete() - service_deletion.wait_for_finish(timeout=600) + try: + service_deletion.wait_for_finish(timeout=settings.ORCHESTRATION_REQUEST_TIMEOUT) + except TimeoutError: + self._logger.error("Service deletion %s timed out", self._service_instance_name) + raise onap_test_exceptions.ServiceCleanupException if service_deletion.finished: self._logger.info("Service %s deleted", self._service_instance_name) else: diff --git a/src/onaptests/steps/instantiate/vf_module_ala_carte.py b/src/onaptests/steps/instantiate/vf_module_ala_carte.py index 2c1834e..ee20bc6 100644 --- a/src/onaptests/steps/instantiate/vf_module_ala_carte.py +++ b/src/onaptests/steps/instantiate/vf_module_ala_carte.py @@ -1,4 +1,3 @@ -import time from typing import Iterable from uuid import uuid4 from yaml import load @@ -143,10 +142,14 @@ class YamlTemplateVfModuleAlaCarteInstantiateStep(YamlTemplateBaseStep): tenant, self._service_instance_name, vnf_parameters=self.get_vnf_parameters(vnf_instance.vnf.name)) - while not vf_module_instantiation.finished: - time.sleep(10) - if vf_module_instantiation.failed: - raise onap_test_exceptions.VfModuleInstantiateException + try: + vf_module_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + if vf_module_instantiation.failed: + self._logger.error("VfModule instantiation %s failed", vf_module.name) + raise onap_test_exceptions.VfModuleInstantiateException + except TimeoutError: + self._logger.error("VfModule instantiation %s timed out", vf_module.name) + raise onap_test_exceptions.VfModuleInstantiateException @YamlTemplateBaseStep.store_state(cleanup=True) def cleanup(self) -> None: @@ -164,16 +167,14 @@ class YamlTemplateVfModuleAlaCarteInstantiateStep(YamlTemplateBaseStep): self._logger.info("Delete VF Module %s", vf_module.name) vf_module_deletion = vf_module.delete() - nb_try = 0 - nb_try_max = 30 - - while not vf_module_deletion.finished and nb_try < nb_try_max: - self._logger.info("Wait for vf module deletion") - nb_try += 1 - time.sleep(20) - if vf_module_deletion.finished: + + try: + vf_module_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + if vf_module_deletion.failed: + self._logger.error("VfModule deletion %s failed", vf_module.name) + raise onap_test_exceptions.VfModuleCleanupException self._logger.info("VfModule %s deleted", vf_module.name) - else: - self._logger.error("VfModule deletion %s failed", vf_module.name) + except TimeoutError: + self._logger.error("VfModule deletion %s timed out", vf_module.name) raise onap_test_exceptions.VfModuleCleanupException super().cleanup() diff --git a/src/onaptests/steps/instantiate/vl_ala_carte.py b/src/onaptests/steps/instantiate/vl_ala_carte.py index f9ac560..fd040aa 100644 --- a/src/onaptests/steps/instantiate/vl_ala_carte.py +++ b/src/onaptests/steps/instantiate/vl_ala_carte.py @@ -1,4 +1,3 @@ -import time import re from typing import Iterable from uuid import uuid4 @@ -141,9 +140,13 @@ class YamlTemplateVlAlaCarteInstantiateStep(YamlTemplateBaseStep): platform, network_instance_name=f"{self.service_instance_name}_net_{idx}", subnets=self.get_subnets(network.name)) - while not net_instantiation.finished: - time.sleep(10) - if net_instantiation.failed: + try: + net_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + if net_instantiation.failed: + self._logger.error("VL instantiation %s failed", net_instantiation.name) + raise onap_test_exceptions.NetworkInstantiateException + except TimeoutError: + self._logger.error("VL instantiation %s timed out", net_instantiation.name) raise onap_test_exceptions.NetworkInstantiateException @YamlTemplateBaseStep.store_state(cleanup=True) @@ -158,7 +161,11 @@ class YamlTemplateVlAlaCarteInstantiateStep(YamlTemplateBaseStep): self._logger.info("Start network deletion %s",net_instance.name) net_deletion = net_instance.delete() try: - net_deletion.wait_for_finish() + net_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + if net_deletion.failed: + self._logger.error("VL deletion %s failed", net_instance.name) + raise onap_test_exceptions.NetworkCleanupException except TimeoutError: + self._logger.error("VL deletion %s timed out", net_instance.name) raise onap_test_exceptions.NetworkCleanupException super().cleanup() diff --git a/src/onaptests/steps/instantiate/vnf_ala_carte.py b/src/onaptests/steps/instantiate/vnf_ala_carte.py index 64ea090..9dc062a 100644 --- a/src/onaptests/steps/instantiate/vnf_ala_carte.py +++ b/src/onaptests/steps/instantiate/vnf_ala_carte.py @@ -1,4 +1,3 @@ -import time from uuid import uuid4 from yaml import load @@ -118,9 +117,13 @@ class YamlTemplateVnfAlaCarteInstantiateStep(YamlTemplateBaseStep): cloud_region, tenant, f"{self.service_instance_name}_vnf_{idx}") - while not vnf_instantiation.finished: - time.sleep(10) - if vnf_instantiation.failed: + try: + vnf_instantiation.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + if vnf_instantiation.failed: + self._logger.error("VNF instantiation %s failed", vnf.name) + raise onap_test_exceptions.VnfInstantiateException + except TimeoutError: + self._logger.error("VNF instantiation %s timed out", vnf.name) raise onap_test_exceptions.VnfInstantiateException @YamlTemplateBaseStep.store_state(cleanup=True) @@ -133,16 +136,13 @@ class YamlTemplateVnfAlaCarteInstantiateStep(YamlTemplateBaseStep): """ for vnf_instance in self._service_instance.vnf_instances: vnf_deletion = vnf_instance.delete() - nb_try = 0 - nb_try_max = 30 - - while not vnf_deletion.finished and nb_try < nb_try_max: - self._logger.info("Wait for vnf deletion") - nb_try += 1 - time.sleep(15) - if vnf_deletion.finished: - self._logger.info("VNF %s deleted", vnf_instance.name) - else: - self._logger.error("VNF deletion %s failed", vnf_instance.name) + + try: + vnf_deletion.wait_for_finish(settings.ORCHESTRATION_REQUEST_TIMEOUT) + if vnf_deletion.failed: + self._logger.error("VNF deletion %s failed", vnf_instance.name) + raise onap_test_exceptions.VnfCleanupException + except TimeoutError: + self._logger.error("VNF deletion %s timed out", vnf_instance.name) raise onap_test_exceptions.VnfCleanupException super().cleanup() -- cgit 1.2.3-korg