2.4.2 policy-handler - fixed race on step-timer

- fixed the bug of unpredictably stopping of the periodic catch-up
   step-timer due to thread race condition in policy-handler
  = added critical sections under the reentrant lock on every group
      of local var change in step-timer
- added more stats for healthcheck to track each type of
   job-operation separately
   = that helps narrowing down identifying the potential problems
- unit test coverage 76%

Change-Id: I92ddf6c92a3d225d9b87427e3edfb7f80669501a
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-472

1 files changed, 7 insertions, 5 deletions

diff --git a/policyhandler/policy_updater.py b/policyhandler/policy_updater.py
index 0b9e227..70823fa 100644
--- a/policyhandler/policy_updater.py
+++ b/policyhandler/policy_updater.py
@@ -47,7 +47,7 @@ class PolicyUpdater(Thread):
         self._aud_shutdown = None
         self._aud_catch_up = None
 
-        catch_up_config = Config.config.get("catch_up", {})
+        catch_up_config = Config.config.get(CATCH_UP, {})
         self._catch_up_interval = catch_up_config.get("interval") or 15*60
         self._catch_up_max_skips = catch_up_config.get("max_skips") or 3
         self._catch_up_skips = 0
@@ -120,8 +120,10 @@ class PolicyUpdater(Thread):
         """need to bring the latest policies to DCAE-Controller"""
         with self._lock:
             self._aud_catch_up = audit or Audit(req_message=AUTO_CATCH_UP)
-            PolicyUpdater._logger.info("catch_up %s request_id %s",
-                self._aud_catch_up.req_message, self._aud_catch_up.request_id)
+            PolicyUpdater._logger.info(
+                "catch_up %s request_id %s",
+                self._aud_catch_up.req_message, self._aud_catch_up.request_id
+            )
 
         self.enqueue()
 
@@ -131,8 +133,8 @@ class PolicyUpdater(Thread):
             return
 
         if self._catch_up_timer:
-            self._catch_up_timer.next()
             self._logger.info("next step catch_up_timer in %s", self._catch_up_interval)
+            self._catch_up_timer.next()
             return
 
         self._catch_up_timer = StepTimer(
@@ -142,8 +144,8 @@ class PolicyUpdater(Thread):
             PolicyUpdater._logger,
             self
         )
-        self._catch_up_timer.start()
         self._logger.info("started catch_up_timer in %s", self._catch_up_interval)
+        self._catch_up_timer.start()
 
     def _pause_catch_up_timer(self):
         """pause catch_up_timer"""