diff options
author | Alex Shatov <alexs@att.com> | 2018-05-10 09:23:16 -0400 |
---|---|---|
committer | Alex Shatov <alexs@att.com> | 2018-05-10 09:23:16 -0400 |
commit | f2d7bef13705812c1bf147c2fb65162fbf385c6b (patch) | |
tree | e6efb8e25287576a48952942aacdb3cf84a825ff /policyhandler/step_timer.py | |
parent | 50bed534083c96cbf1f8fa4e220cb2b00dff9621 (diff) |
2.4.3 policy-handler - try-catch top Exceptions
- added try-except for top level Exception into all threads
of policy-handler to avoid losing the thread and tracking
the unexpected crashes
- rediscover the deployment-handler if not found before
and after each catchup
- refactored audit - separated metrics from audit
- added more stats and runtime info to healthcheck
= gc counts and garbage info if any detected
= memory usage - to detect the potential memory leaks
= request_id to all stats
= stats of active requests
- avoid reallocating the whole Queue of policy-updates after catchup
= clear of the internal queue under proper lock
Change-Id: I3fabcaac70419a68bd070ff7d591a75942f37663
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-483
Diffstat (limited to 'policyhandler/step_timer.py')
-rw-r--r-- | policyhandler/step_timer.py | 27 |
1 files changed, 17 insertions, 10 deletions
diff --git a/policyhandler/step_timer.py b/policyhandler/step_timer.py index 3936107..2a13dd5 100644 --- a/policyhandler/step_timer.py +++ b/policyhandler/step_timer.py @@ -18,6 +18,7 @@ """periodically callback""" +import json from datetime import datetime from threading import Event, RLock, Thread @@ -51,11 +52,11 @@ class StepTimer(Thread): self._request = StepTimer.INIT self._req_count = 0 self._req_time = 0 - self._req_ts = datetime.now() + self._req_ts = datetime.utcnow() self._substep = None self._substep_time = 0 - self._substep_ts = datetime.now() + self._substep_ts = datetime.utcnow() def get_timer_status(self): """returns timer status""" @@ -104,9 +105,9 @@ class StepTimer(Thread): prev_req = self._request self._request = request - now = datetime.now() - self._req_time = (now - self._req_ts).total_seconds() - self._req_ts = now + utcnow = datetime.utcnow() + self._req_time = (utcnow - self._req_ts).total_seconds() + self._req_ts = utcnow self._logger.info("{0}[{1}] {2}->{3}".format( self.name, self._req_time, prev_req, self.get_timer_status())) @@ -114,9 +115,9 @@ class StepTimer(Thread): """log exe step""" with self._lock: self._substep = substep - now = datetime.now() - self._substep_time = (now - self._substep_ts).total_seconds() - self._substep_ts = now + utcnow = datetime.utcnow() + self._substep_time = (utcnow - self._substep_ts).total_seconds() + self._substep_ts = utcnow self._logger.info("[{0}] {1}".format(self._substep_time, self.get_timer_status())) def run(self): @@ -147,8 +148,14 @@ class StepTimer(Thread): if self._paused: self._timer_substep("paused - skip on_time event") else: - self._timer_substep("on_time event") - self._on_time(*self._args, **self._kwargs) + try: + self._timer_substep("on_time event") + self._on_time(*self._args, **self._kwargs) + except Exception as ex: + error_msg = ("{0}: crash {1} {2} at {3}: args({4}), kwargs({5})" + .format(self.name, type(ex).__name__, str(ex), "_on_time", + json.dumps(self._args), json.dumps(self._kwargs))) + self._logger.exception(error_msg) self._timer_substep("waiting for next...") self._next.wait() |