2.4.2 policy-handler - fixed race on step-timer

- fixed the bug of unpredictably stopping of the periodic catch-up
   step-timer due to thread race condition in policy-handler
  = added critical sections under the reentrant lock on every group
      of local var change in step-timer
- added more stats for healthcheck to track each type of
   job-operation separately
   = that helps narrowing down identifying the potential problems
- unit test coverage 76%

Change-Id: I92ddf6c92a3d225d9b87427e3edfb7f80669501a
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-472

1 files changed, 10 insertions, 6 deletions

diff --git a/policyhandler/web_server.py b/policyhandler/web_server.py
index 20e74e3..e9cc9cc 100644
--- a/policyhandler/web_server.py
+++ b/policyhandler/web_server.py
@@ -56,7 +56,8 @@ class _PolicyWeb(object):
     def policy_latest(self, policy_id):
         """retireves the latest policy identified by policy_id"""
         req_info = _PolicyWeb._get_request_info(cherrypy.request)
-        audit = Audit(req_message=req_info, headers=cherrypy.request.headers)
+        audit = Audit(job_name="get_latest_policy",
+                      req_message=req_info, headers=cherrypy.request.headers)
         PolicyWeb.logger.info("%s policy_id=%s headers=%s", \
             req_info, policy_id, json.dumps(cherrypy.request.headers))
 
@@ -74,7 +75,8 @@ class _PolicyWeb(object):
     def _get_all_policies_latest(self):
         """retireves all the latest policies on GET /policies_latest"""
         req_info = _PolicyWeb._get_request_info(cherrypy.request)
-        audit = Audit(req_message=req_info, headers=cherrypy.request.headers)
+        audit = Audit(job_name="get_all_policies_latest",
+                      req_message=req_info, headers=cherrypy.request.headers)
 
         PolicyWeb.logger.info("%s", req_info)
 
@@ -146,7 +148,8 @@ class _PolicyWeb(object):
         str_policy_filter = json.dumps(policy_filter)
 
         req_info = _PolicyWeb._get_request_info(cherrypy.request)
-        audit = Audit(req_message="{0}: {1}".format(req_info, str_policy_filter), \
+        audit = Audit(job_name="get_latest_policies",
+                      req_message="{0}: {1}".format(req_info, str_policy_filter), \
             headers=cherrypy.request.headers)
         PolicyWeb.logger.info("%s: policy_filter=%s headers=%s", \
             req_info, str_policy_filter, json.dumps(cherrypy.request.headers))
@@ -168,7 +171,7 @@ class _PolicyWeb(object):
         """catch up with all DCAE policies"""
         started = str(datetime.now())
         req_info = _PolicyWeb._get_request_info(cherrypy.request)
-        audit = Audit(req_message=req_info, headers=cherrypy.request.headers)
+        audit = Audit(job_name="catch_up", req_message=req_info, headers=cherrypy.request.headers)
 
         PolicyWeb.logger.info("%s", req_info)
         PolicyReceiver.catch_up(audit)
@@ -182,7 +185,7 @@ class _PolicyWeb(object):
     def shutdown(self):
         """Shutdown the policy-handler"""
         req_info = _PolicyWeb._get_request_info(cherrypy.request)
-        audit = Audit(req_message=req_info, headers=cherrypy.request.headers)
+        audit = Audit(job_name="shutdown", req_message=req_info, headers=cherrypy.request.headers)
 
         PolicyWeb.logger.info("%s: --- stopping REST API of policy-handler ---", req_info)
 
@@ -203,7 +206,8 @@ class _PolicyWeb(object):
     def healthcheck(self):
         """returns the healthcheck results"""
         req_info = _PolicyWeb._get_request_info(cherrypy.request)
-        audit = Audit(req_message=req_info, headers=cherrypy.request.headers)
+        audit = Audit(job_name="healthcheck",
+                      req_message=req_info, headers=cherrypy.request.headers)
 
         PolicyWeb.logger.info("%s", req_info)