2.4.3 policy-handler - try-catch top Exceptions

- added try-except for top level Exception into all threads
  of policy-handler to avoid losing the thread and tracking
  the unexpected crashes
- rediscover the deployment-handler if not found before
  and after each catchup
- refactored audit - separated metrics from audit
- added more stats and runtime info to healthcheck
  = gc counts and garbage info if any detected
  = memory usage - to detect the potential memory leaks
  = request_id to all stats
  = stats of active requests
- avoid reallocating the whole Queue of policy-updates after catchup
  = clear of the internal queue under proper lock

Change-Id: I3fabcaac70419a68bd070ff7d591a75942f37663
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-483

1 files changed, 5 insertions, 6 deletions

diff --git a/policyhandler/web_server.py b/policyhandler/web_server.py
index e9cc9cc..5314791 100644
--- a/policyhandler/web_server.py
+++ b/policyhandler/web_server.py
@@ -169,7 +169,7 @@ class _PolicyWeb(object):
     @cherrypy.tools.json_out()
     def catch_up(self):
         """catch up with all DCAE policies"""
-        started = str(datetime.now())
+        started = str(datetime.utcnow())
         req_info = _PolicyWeb._get_request_info(cherrypy.request)
         audit = Audit(job_name="catch_up", req_message=req_info, headers=cherrypy.request.headers)
 
@@ -193,11 +193,10 @@ class _PolicyWeb(object):
 
         PolicyReceiver.shutdown(audit)
 
-        health = json.dumps(Audit.health())
-        audit.info("policy_handler health: {0}".format(health))
-        PolicyWeb.logger.info("policy_handler health: %s", health)
+        PolicyWeb.logger.info("policy_handler health: {0}"
+                              .format(json.dumps(audit.health(full=True))))
         PolicyWeb.logger.info("%s: --------- the end -----------", req_info)
-        res = str(datetime.now())
+        res = str(datetime.utcnow())
         audit.info_requested(res)
         return "goodbye! shutdown requested {0}".format(res)
 
@@ -211,7 +210,7 @@ class _PolicyWeb(object):
 
         PolicyWeb.logger.info("%s", req_info)
 
-        res = Audit.health()
+        res = audit.health()
 
         PolicyWeb.logger.info("healthcheck %s: res=%s", req_info, json.dumps(res))