summaryrefslogtreecommitdiffstats
path: root/policyhandler/deploy_handler.py
diff options
context:
space:
mode:
authorAlex Shatov <alexs@att.com>2018-05-10 09:23:16 -0400
committerAlex Shatov <alexs@att.com>2018-05-10 09:23:16 -0400
commitf2d7bef13705812c1bf147c2fb65162fbf385c6b (patch)
treee6efb8e25287576a48952942aacdb3cf84a825ff /policyhandler/deploy_handler.py
parent50bed534083c96cbf1f8fa4e220cb2b00dff9621 (diff)
2.4.3 policy-handler - try-catch top Exceptions
- added try-except for top level Exception into all threads of policy-handler to avoid losing the thread and tracking the unexpected crashes - rediscover the deployment-handler if not found before and after each catchup - refactored audit - separated metrics from audit - added more stats and runtime info to healthcheck = gc counts and garbage info if any detected = memory usage - to detect the potential memory leaks = request_id to all stats = stats of active requests - avoid reallocating the whole Queue of policy-updates after catchup = clear of the internal queue under proper lock Change-Id: I3fabcaac70419a68bd070ff7d591a75942f37663 Signed-off-by: Alex Shatov <alexs@att.com> Issue-ID: DCAEGEN2-483
Diffstat (limited to 'policyhandler/deploy_handler.py')
-rw-r--r--policyhandler/deploy_handler.py85
1 files changed, 47 insertions, 38 deletions
diff --git a/policyhandler/deploy_handler.py b/policyhandler/deploy_handler.py
index 139e660..4ea5ad1 100644
--- a/policyhandler/deploy_handler.py
+++ b/policyhandler/deploy_handler.py
@@ -26,7 +26,7 @@ import requests
from .config import Config
from .customize import CustomizerUser
from .discovery import DiscoveryClient
-from .onap.audit import REQUEST_X_ECOMP_REQUESTID, Audit, AuditHttpCode
+from .onap.audit import REQUEST_X_ECOMP_REQUESTID, AuditHttpCode, Metrics
POOL_SIZE = 1
@@ -44,27 +44,27 @@ class DeployHandler(object):
_server_instance_uuid = None
@staticmethod
- def _lazy_init(audit):
+ def _lazy_init(audit, rediscover=False):
""" set static properties """
- if DeployHandler._lazy_inited:
+ if DeployHandler._lazy_inited and not rediscover:
return
- DeployHandler._lazy_inited = True
- DeployHandler._custom_kwargs = CustomizerUser.get_customizer() \
- .get_deploy_handler_kwargs(audit)
- if not DeployHandler._custom_kwargs \
- or not isinstance(DeployHandler._custom_kwargs, dict):
+ DeployHandler._custom_kwargs = (CustomizerUser.get_customizer()
+ .get_deploy_handler_kwargs(audit))
+ if (not DeployHandler._custom_kwargs
+ or not isinstance(DeployHandler._custom_kwargs, dict)):
DeployHandler._custom_kwargs = {}
- DeployHandler._requests_session = requests.Session()
- DeployHandler._requests_session.mount(
- 'https://',
- requests.adapters.HTTPAdapter(pool_connections=POOL_SIZE, pool_maxsize=POOL_SIZE)
- )
- DeployHandler._requests_session.mount(
- 'http://',
- requests.adapters.HTTPAdapter(pool_connections=POOL_SIZE, pool_maxsize=POOL_SIZE)
- )
+ if not DeployHandler._requests_session:
+ DeployHandler._requests_session = requests.Session()
+ DeployHandler._requests_session.mount(
+ 'https://',
+ requests.adapters.HTTPAdapter(pool_connections=POOL_SIZE, pool_maxsize=POOL_SIZE)
+ )
+ DeployHandler._requests_session.mount(
+ 'http://',
+ requests.adapters.HTTPAdapter(pool_connections=POOL_SIZE, pool_maxsize=POOL_SIZE)
+ )
config_dh = Config.config.get("deploy_handler")
if config_dh and isinstance(config_dh, dict):
@@ -77,7 +77,7 @@ class DeployHandler(object):
DeployHandler._target_entity = config_dh.get("target_entity", "deployment_handler")
DeployHandler._url = config_dh.get("url")
DeployHandler._logger.info("dns based routing to %s: url(%s)",
- DeployHandler._target_entity, DeployHandler._url)
+ DeployHandler._target_entity, DeployHandler._url)
if not DeployHandler._url:
# discover routing to deployment-handler at consul-services
@@ -85,14 +85,18 @@ class DeployHandler(object):
# config for policy-handler <= 2.3.1
# "deploy_handler" : "deployment_handler"
DeployHandler._target_entity = str(config_dh or "deployment_handler")
- DeployHandler._url = DiscoveryClient.get_service_url(audit, DeployHandler._target_entity)
+ DeployHandler._url = DiscoveryClient.get_service_url(audit,
+ DeployHandler._target_entity)
DeployHandler._url_policy = str(DeployHandler._url or "") + '/policy'
DeployHandler._logger.info(
"got %s policy url(%s)", DeployHandler._target_entity, DeployHandler._url_policy)
+ DeployHandler._lazy_inited = bool(DeployHandler._url)
+
+
@staticmethod
- def policy_update(audit, message):
+ def policy_update(audit, message, rediscover=False):
"""
post policy_updated message to deploy-handler
@@ -101,10 +105,10 @@ class DeployHandler(object):
if not message:
return
- DeployHandler._lazy_init(audit)
- sub_aud = Audit(aud_parent=audit, targetEntity=DeployHandler._target_entity,
- targetServiceName=DeployHandler._url_policy)
- headers = {REQUEST_X_ECOMP_REQUESTID : sub_aud.request_id}
+ DeployHandler._lazy_init(audit, rediscover)
+ metrics = Metrics(aud_parent=audit, targetEntity=DeployHandler._target_entity,
+ targetServiceName=DeployHandler._url_policy)
+ headers = {REQUEST_X_ECOMP_REQUESTID : metrics.request_id}
msg_str = json.dumps(message)
headers_str = json.dumps(headers)
@@ -114,14 +118,14 @@ class DeployHandler(object):
log_data = " msg={0} headers={1}".format(msg_str, headers_str)
log_line = log_action + log_data
DeployHandler._logger.info(log_line)
- sub_aud.metrics_start(log_line)
+ metrics.metrics_start(log_line)
if not DeployHandler._url:
error_msg = "no url found to {0}".format(log_line)
DeployHandler._logger.error(error_msg)
- sub_aud.set_http_status_code(AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value)
+ metrics.set_http_status_code(AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value)
audit.set_http_status_code(AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value)
- sub_aud.metrics(error_msg)
+ metrics.metrics(error_msg)
return
res = None
@@ -130,25 +134,30 @@ class DeployHandler(object):
DeployHandler._url_policy, json=message, headers=headers,
**DeployHandler._custom_kwargs
)
- except requests.exceptions.RequestException as ex:
- error_msg = "failed to {0}: {1}{2}".format(log_action, str(ex), log_data)
+ except Exception as ex:
+ error_code = (AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value
+ if isinstance(ex, requests.exceptions.RequestException)
+ else AuditHttpCode.SERVER_INTERNAL_ERROR.value)
+ error_msg = ("failed to {0} {1}: {2}{3}"
+ .format(log_action, type(ex).__name__, str(ex), log_data))
DeployHandler._logger.exception(error_msg)
- sub_aud.set_http_status_code(AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value)
- audit.set_http_status_code(AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value)
- sub_aud.metrics(error_msg)
+ metrics.set_http_status_code(error_code)
+ audit.set_http_status_code(error_code)
+ metrics.metrics(error_msg)
return
- sub_aud.set_http_status_code(res.status_code)
+ metrics.set_http_status_code(res.status_code)
audit.set_http_status_code(res.status_code)
log_line = "response {0} from {1}: text={2}{3}" \
.format(res.status_code, log_action, res.text, log_data)
- sub_aud.metrics(log_line)
- DeployHandler._logger.info(log_line)
+ metrics.metrics(log_line)
if res.status_code != requests.codes.ok:
+ DeployHandler._logger.error(log_line)
return
+ DeployHandler._logger.info(log_line)
result = res.json() or {}
prev_server_instance_uuid = DeployHandler._server_instance_uuid
DeployHandler._server_instance_uuid = result.get("server_instance_uuid")
@@ -156,9 +165,9 @@ class DeployHandler(object):
deployment_handler_changed = (prev_server_instance_uuid
and prev_server_instance_uuid != DeployHandler._server_instance_uuid)
if deployment_handler_changed:
- log_line = "deployment_handler_changed: {1} != {0}" \
- .format(prev_server_instance_uuid, DeployHandler._server_instance_uuid)
- sub_aud.info(log_line)
+ log_line = ("deployment_handler_changed: {1} != {0}"
+ .format(prev_server_instance_uuid, DeployHandler._server_instance_uuid))
+ metrics.info(log_line)
DeployHandler._logger.info(log_line)
return deployment_handler_changed