4.2.0 policy-handler - periodic reconfigure

- reconfigure == periodically retrieve the policy-handler config
  from consul-kv and compare to previous config and subconfigs.
  If changed, reconfigure the subunits
- selectively change one or any settings for the following
  = catch_up timer interval
  = reconfigure timer interval
  = deployment-handler url and params (thread-safe)
  = policy-engine url and params (thread-safe)
  = web-socket url to policy-engine (through a callback)
- each subunit has its own Settings that keep track of changes
- try-catch and metrics around discovery - consul API
- hidden the secrets from logs
- froze the web-socket version to 0.49.0 because 0.50.0
  and 0.51.0 are broken - looking around for stable alternatives
- fixed-adapted the callbacks passed to the web-socket lib
  that changed its API in 0.49.0 and later
- log the stack on the exception occurring in the web-socket lib
- unit test refactoring

Change-Id: Id53bad59660a197f59d9aeb7c05ab761d1060cd0
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-470

1 files changed, 84 insertions, 35 deletions

diff --git a/policyhandler/discovery.py b/policyhandler/discovery.py
index ce24c3d..4e6bc3d 100644
--- a/policyhandler/discovery.py
+++ b/policyhandler/discovery.py
@@ -25,6 +25,8 @@ import logging
 import requests
 
 from .customize import CustomizerUser
+from .onap.audit import AuditHttpCode, Metrics
+
 
 class DiscoveryClient(object):
     """talking to consul at http://consul:8500
@@ -40,56 +42,103 @@ class DiscoveryClient(object):
         -p <outport>:<innerport>
         ${APPNAME}:latest
     """
+    CONSUL_ENTITY = "consul"
     CONSUL_SERVICE_MASK = "http://consul:8500/v1/catalog/service/{0}"
     CONSUL_KV_MASK = "http://consul:8500/v1/kv/{0}"
     _logger = logging.getLogger("policy_handler.discovery")
 
     @staticmethod
-    def get_service_url(audit, service_name):
+    def _discover_service(audit, service_name, service_path):
         """find the service record in consul"""
-        service_path = DiscoveryClient.CONSUL_SERVICE_MASK.format(service_name)
-        log_line = "discover {0}".format(service_path)
-        DiscoveryClient._logger.info(log_line)
-        audit.info(log_line)
         response = requests.get(service_path)
-
-        log_line = "response {0} for {1}: {2}".format(
-            response.status_code, service_path, response.text)
-        DiscoveryClient._logger.info(log_line)
-        audit.info(log_line)
+        DiscoveryClient._logger.info(audit.info("response {} from {}: {}".format(
+            response.status_code, service_path, response.text)))
 
         response.raise_for_status()
+        status_code = response.status_code
+        service = response.json()[0]
+        return (status_code,
+                CustomizerUser.get_customizer().get_service_url(audit, service_name, service))
+
+    @staticmethod
+    def get_service_url(audit, service_name):
+        """find the service record in consul"""
+        service_path = DiscoveryClient.CONSUL_SERVICE_MASK.format(service_name)
+        metrics = Metrics(aud_parent=audit, targetEntity=DiscoveryClient.CONSUL_ENTITY,
+                          targetServiceName=service_path)
+
+        log_line = "get from {} at {}".format(DiscoveryClient.CONSUL_ENTITY, service_path)
 
-        service = response.json()
-        if not service:
-            log_line = "failed discover {0}".format(service_path)
-            DiscoveryClient._logger.error(log_line)
-            audit.error(log_line)
-            return
-        service = service[0]
+        DiscoveryClient._logger.info(metrics.metrics_start(log_line))
+        status_code = None
+        try:
+            (status_code,
+             service_url) = DiscoveryClient._discover_service(audit, service_name, service_path)
+        except Exception as ex:
+            error_code = (AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value
+                          if isinstance(ex, requests.exceptions.RequestException)
+                          else AuditHttpCode.SERVER_INTERNAL_ERROR.value)
+            error_msg = ("failed {}/{} to {} {}: {}".format(status_code, error_code, log_line,
+                                                            type(ex).__name__, str(ex)))
+            DiscoveryClient._logger.exception(error_msg)
+            metrics.set_http_status_code(error_code)
+            audit.set_http_status_code(error_code)
+            metrics.metrics(error_msg)
+            return None
 
-        service_url = CustomizerUser.get_customizer().get_service_url(audit, service_name, service)
         if not service_url:
-            log_line = "failed to get service_url for {0}".format(service_name)
-            DiscoveryClient._logger.error(log_line)
-            audit.error(log_line)
-            return
-
-        log_line = "got service_url: {0} for {1}".format(service_url, service_name)
-        DiscoveryClient._logger.info(log_line)
-        audit.info(log_line)
+            error_code = AuditHttpCode.DATA_ERROR.value
+            error_msg = "failed {}/{} to {}".format(status_code, error_code, log_line)
+            DiscoveryClient._logger.error(audit.error(error_msg))
+            metrics.set_http_status_code(error_code)
+            audit.set_http_status_code(error_code)
+            metrics.metrics(error_msg)
+            return None
+
+        log_line = "response {} {}".format(status_code, log_line)
+        DiscoveryClient._logger.info(audit.info("got service_url: {} after {}"
+                                                .format(service_url, log_line)))
+
+        metrics.set_http_status_code(status_code)
+        audit.set_http_status_code(status_code)
+        metrics.metrics(log_line)
         return service_url
 
     @staticmethod
-    def get_value(key):
-        """get the value for the key from consul-kv"""
-        response = requests.get(DiscoveryClient.CONSUL_KV_MASK.format(key))
+    def _get_value_from_kv(url):
+        """get the value from consul-kv at discovery url"""
+        response = requests.get(url)
         response.raise_for_status()
         data = response.json()
-        if not data:
-            DiscoveryClient._logger.error("failed get_value %s", key)
-            return
         value = base64.b64decode(data[0]["Value"]).decode("utf-8")
-        DiscoveryClient._logger.info("consul-kv key=%s value(%s) data=%s",
-                                     key, value, json.dumps(data))
-        return json.loads(value)
+        return response.status_code, json.loads(value)
+
+    @staticmethod
+    def get_value(audit, key):
+        """get the value for the key from consul-kv"""
+        discovery_url = DiscoveryClient.CONSUL_KV_MASK.format(key)
+        metrics = Metrics(aud_parent=audit, targetEntity=DiscoveryClient.CONSUL_ENTITY,
+                          targetServiceName=discovery_url)
+
+        log_line = "get from {} at {}".format(DiscoveryClient.CONSUL_ENTITY, discovery_url)
+
+        DiscoveryClient._logger.info(metrics.metrics_start(log_line))
+        try:
+            status_code, value = DiscoveryClient._get_value_from_kv(discovery_url)
+        except Exception as ex:
+            error_code = (AuditHttpCode.SERVICE_UNAVAILABLE_ERROR.value
+                          if isinstance(ex, requests.exceptions.RequestException)
+                          else AuditHttpCode.SERVER_INTERNAL_ERROR.value)
+            error_msg = ("failed {}/{} to {} {}: {}".format(status_code, error_code, log_line,
+                                                            type(ex).__name__, str(ex)))
+            DiscoveryClient._logger.exception(error_msg)
+            metrics.set_http_status_code(error_code)
+            audit.set_http_status_code(error_code)
+            metrics.metrics(error_msg)
+            return None
+
+        log_line = "response {} {}".format(status_code, log_line)
+        metrics.set_http_status_code(status_code)
+        audit.set_http_status_code(status_code)
+        metrics.metrics(log_line)
+        return value