diff options
Diffstat (limited to 'snmptrap/mod/trapd_stormwatch.py')
-rw-r--r-- | snmptrap/mod/trapd_stormwatch.py | 199 |
1 files changed, 107 insertions, 92 deletions
diff --git a/snmptrap/mod/trapd_stormwatch.py b/snmptrap/mod/trapd_stormwatch.py index 9e41bd0..4c374bb 100644 --- a/snmptrap/mod/trapd_stormwatch.py +++ b/snmptrap/mod/trapd_stormwatch.py @@ -1,5 +1,5 @@ # ============LICENSE_START======================================================= -# Copyright (c) 2020 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2020-2021 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ threshold (storm), and if so it will return "False" so the trap can be logged and immediately discarded """ -__docformat__ = 'restructuredtext' +__docformat__ = "restructuredtext" import sys import os @@ -98,11 +98,11 @@ def sw_clear_dicts(): sws.sw_config_category.clear() return True except Exception as e: - msg = "unable to reset stormwatch dictionaries - results will be indeterminate: %s" % ( - e) + msg = "unable to reset stormwatch dictionaries - results will be indeterminate: %s" % (e) ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_WARN, tds.CODE_GENERAL, msg) return False + # # # # # # # # # # # # # # fx: sw_load_trap_config # - load trap configurations from CBS response @@ -121,10 +121,10 @@ def sw_load_trap_config(_config): try: sws.sw_storm_active_dict ret = sw_clear_dicts() - msg = ("reset existing sws dictionaries to empty") + msg = "reset existing sws dictionaries to empty" ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) except NameError: - msg = ("sws dictionaries not present - initializing") + msg = "sws dictionaries not present - initializing" ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) ret = sw_init() @@ -134,24 +134,22 @@ def sw_load_trap_config(_config): # get metric % threshold for logging trap count by-agent to metric log try: stats.metric_log_notification_threshold_pct = int( - _config["trap_config"]["metric_log_notification_threshold_pct"]) - msg = ("metric_log_notification_threshold_pct value: %d" % - stats.metric_log_notification_threshold_pct) + _config["trap_config"]["metric_log_notification_threshold_pct"] + ) + msg = "metric_log_notification_threshold_pct value: %d" % stats.metric_log_notification_threshold_pct ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) except Exception as e: - msg = ( - "metric_log_notification_threshold_pct not present in config - default to 25") + msg = "metric_log_notification_threshold_pct not present in config - default to 25" ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, tds.CODE_GENERAL, msg) stats.metric_log_notification_threshold_pct = 25 # get stormwatch interval; default to 60 seconds try: - sws.sw_interval_in_seconds = int( - _config["trap_config"]["sw_interval_in_seconds"]) - msg = ("sw_interval_in_seconds value: %d" % sws.sw_interval_in_seconds) + sws.sw_interval_in_seconds = int(_config["trap_config"]["sw_interval_in_seconds"]) + msg = "sw_interval_in_seconds value: %d" % sws.sw_interval_in_seconds ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) except Exception as e: - msg = ("sw_interval_in_seconds not present in config - default to 60 seconds") + msg = "sw_interval_in_seconds not present in config - default to 60 seconds" ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, tds.CODE_GENERAL, msg) sws.sw_interval_in_seconds = 60 @@ -159,7 +157,7 @@ def sw_load_trap_config(_config): try: notify_oids = _config["trap_config"]["notify_oids"] except Exception as e: - msg = ("no trap_config or notify_oids defined") + msg = "no trap_config or notify_oids defined" ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, tds.CODE_GENERAL, msg) return False @@ -167,65 +165,66 @@ def sw_load_trap_config(_config): for trap_block in notify_oids: # oid try: - _oid = trap_block['oid'] + _oid = trap_block["oid"] except Exception as e: - msg = ( - "missing oid value in notify_oids - oid section of CBS config - using empty value, disregard entry") - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, - tds.CODE_GENERAL, msg) + msg = "missing oid value in notify_oids - oid section of CBS config - using empty value, disregard entry" + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, tds.CODE_GENERAL, msg) _oid = None # sw_high_water_in_interval try: - _sw_high_water_in_interval = int( - trap_block['sw_high_water_in_interval']) + _sw_high_water_in_interval = int(trap_block["sw_high_water_in_interval"]) except Exception as e: msg = ( - "missing sw_high_water_in_interval value in notify_oids - oid section of CBS config - using empty value") - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, - tds.CODE_GENERAL, msg) + "missing sw_high_water_in_interval value in notify_oids - oid section of CBS config - using empty value" + ) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, tds.CODE_GENERAL, msg) _sw_high_water_in_interval = None # sw_low_water_in_interval try: - _sw_low_water_in_interval = int( - trap_block['sw_low_water_in_interval']) + _sw_low_water_in_interval = int(trap_block["sw_low_water_in_interval"]) except Exception as e: msg = ( - "missing sw_low_water_in_interval value in notify_oids - oid section of CBS config - using empty value") - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, - tds.CODE_GENERAL, msg) + "missing sw_low_water_in_interval value in notify_oids - oid section of CBS config - using empty value" + ) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, tds.CODE_GENERAL, msg) _sw_low_water_in_interval = None # category try: - _category = trap_block['category'] + _category = trap_block["category"] except Exception as e: - msg = ( - "missing category value in notify_oids - oid section of CBS config - using empty value") - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, - tds.CODE_GENERAL, msg) + msg = "missing category value in notify_oids - oid section of CBS config - using empty value" + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_WARN, tds.CODE_GENERAL, msg) _category = None - if (_oid is not None and _category is not None and - _sw_low_water_in_interval is not None and _sw_high_water_in_interval is not None and - _sw_low_water_in_interval < _sw_high_water_in_interval): + if ( + _oid is not None + and _category is not None + and _sw_low_water_in_interval is not None + and _sw_high_water_in_interval is not None + and _sw_low_water_in_interval < _sw_high_water_in_interval + ): # FMDL: Do we actually need sw_config_oid_dict? - msg = ("oid: %s sw_high_water_in_interval: %d sw_low_water_in_interval: %d category: %s" % ( - _oid, _sw_high_water_in_interval, _sw_low_water_in_interval, _category)) - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, - tds.CODE_GENERAL, msg) + msg = "oid: %s sw_high_water_in_interval: %d sw_low_water_in_interval: %d category: %s" % ( + _oid, + _sw_high_water_in_interval, + _sw_low_water_in_interval, + _category, + ) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) sws.sw_config_oid_dict[_oid] = True sws.sw_config_low_water_in_interval_dict[_oid] = _sw_low_water_in_interval sws.sw_config_high_water_in_interval_dict[_oid] = _sw_high_water_in_interval sws.sw_config_category[_oid] = _category trap_block_counter += 1 else: - msg = ("Missing or incorrect value for stormwatch config entry %d: skipping: %s" % ( - trap_block_counter, trap_block)) - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, - tds.CODE_GENERAL, msg) - + msg = "Missing or incorrect value for stormwatch config entry %d: skipping: %s" % ( + trap_block_counter, + trap_block, + ) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) return trap_block_counter @@ -246,17 +245,18 @@ def sw_log_metrics(): :Variables: """ - msg = "total notifications: %d, interval in seconds: %d" % ( - stats.total_notifications, sws.sw_interval_in_seconds) + msg = "total notifications: %d, interval in seconds: %d" % (stats.total_notifications, sws.sw_interval_in_seconds) ecomp_logger(tds.LOG_TYPE_METRICS, tds.SEV_INFO, tds.CODE_GENERAL, msg) # print metrics for total traps and traps-per-second avg # during sample interval avg_traps_per_second = stats.total_notifications / 60 msg = "total traps: %d, interval in seconds: %d, average traps-per-second: %d" % ( - stats.total_notifications, sws.sw_interval_in_seconds, avg_traps_per_second) - ecomp_logger(tds.LOG_TYPE_METRICS, tds.SEV_WARN, - tds.CODE_GENERAL, msg) + stats.total_notifications, + sws.sw_interval_in_seconds, + avg_traps_per_second, + ) + ecomp_logger(tds.LOG_TYPE_METRICS, tds.SEV_WARN, tds.CODE_GENERAL, msg) # print metrics for any agent that represents more than stats.metric_log_notification_threshold_pct # during sample interval @@ -265,9 +265,13 @@ def sw_log_metrics(): p = c / stats.total_notifications * 100 if p > stats.metric_log_notification_threshold_pct: msg = "agent: %s, notifications: %d, interval in seconds: %d, percent of total traps: %d" % ( - k, c, sws.sw_interval_in_seconds, p) - ecomp_logger(tds.LOG_TYPE_METRICS, tds.SEV_WARN, - tds.CODE_GENERAL, msg) + k, + c, + sws.sw_interval_in_seconds, + p, + ) + ecomp_logger(tds.LOG_TYPE_METRICS, tds.SEV_WARN, tds.CODE_GENERAL, msg) + # # # # # # # # # # # # # # fx: stats_increment_counters @@ -339,8 +343,10 @@ def sw_storm_active(_loc_agent, _loc_oid): # if we are at or above stormwatch interval, re-eval and re-set elapsed_time = int(time.time()) - sws.sw_last_stormwatch_dict_analysis if elapsed_time >= sws.sw_interval_in_seconds: - msg = "%d seconds has elapsed since stormwatch dictionary eval (%d second threshold) - check and reset counters " % ( - elapsed_time, sws.sw_interval_in_seconds) + msg = ( + "%d seconds has elapsed since stormwatch dictionary eval (%d second threshold) - check and reset counters " + % (elapsed_time, sws.sw_interval_in_seconds) + ) ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) sw_log_metrics() sw_reset_counter_dict() @@ -349,8 +355,7 @@ def sw_storm_active(_loc_agent, _loc_oid): # that means it's participating in stormwatch, otherwise bail out try: _high_water_val = sws.sw_config_high_water_in_interval_dict[_loc_oid] - msg = "%s present in stormwatch config - high water value: %d" % ( - _loc_oid, _high_water_val) + msg = "%s present in stormwatch config - high water value: %d" % (_loc_oid, _high_water_val) ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) except Exception as e: return False @@ -375,25 +380,32 @@ def sw_storm_active(_loc_agent, _loc_oid): # if we got this far, trap is in stormwatch configs, we've incremented # counter in sw_storm_counter_dict - figure out if we are over limit if sws.sw_storm_counter_dict[_dict_key] > _high_water_val: - print(f"sws.sw_storm_counter_dict[{_dict_key}]({sws.sw_storm_counter_dict[_dict_key]}) > _high_water_val ({_high_water_val})") + print( + f"sws.sw_storm_counter_dict[{_dict_key}]({sws.sw_storm_counter_dict[_dict_key]}) > _high_water_val ({_high_water_val})" + ) _loc_agent = _dict_key.split()[0] _loc_oid = _dict_key.split()[1] msg = "STORM ACTIVE: received %d events (%s) from %s (greater than high water threshold: %d)" % ( - sws.sw_storm_counter_dict[_dict_key], _loc_oid, _loc_agent, _high_water_val) + sws.sw_storm_counter_dict[_dict_key], + _loc_oid, + _loc_agent, + _high_water_val, + ) ecomp_logger(tds.LOG_TYPE_AUDIT, tds.SEV_WARN, tds.CODE_GENERAL, msg) try: sws.sw_storm_active_dict[_dict_key] = True except Exception as e: - msg = "ERROR setting %s in storm active state: %s " % ( - _dict_key, e) - ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_ERROR, - tds.CODE_GENERAL, msg) + msg = "ERROR setting %s in storm active state: %s " % (_dict_key, e) + ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_ERROR, tds.CODE_GENERAL, msg) return True else: - print(f"NOT sws.sw_storm_counter_dict[{_dict_key}]({sws.sw_storm_counter_dict[_dict_key]}) > _high_water_val ({_high_water_val})") + print( + f"NOT sws.sw_storm_counter_dict[{_dict_key}]({sws.sw_storm_counter_dict[_dict_key]}) > _high_water_val ({_high_water_val})" + ) return False + # # # # # # # # # # # # # # fx: sw_reset_counter_dict # - reset counter dictionary on <interval> boundaries @@ -416,11 +428,9 @@ def sw_reset_counter_dict(): # publish stats to MR... try: msg = "publish counts-by-oid from stats.oid_counter_dict" - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, - tds.CODE_GENERAL, msg) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) msg = "publish count-by-agent from stats.agent_counter_dict" - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, - tds.CODE_GENERAL, msg) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) except Exception as e: msg = "unable to publish counts by oid and agent to MR: " % (e) ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_WARN, tds.CODE_GENERAL, msg) @@ -431,8 +441,7 @@ def sw_reset_counter_dict(): stats.agent_counter_dict.clear() stats.total_notifications = 0 except Exception as e: - msg = "unable to reset counts by oid and agent dictionaries - stats will be INNACURATE: " % ( - e) + msg = "unable to reset counts by oid and agent dictionaries - stats will be INNACURATE: " % (e) ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_WARN, tds.CODE_GENERAL, msg) # </stats> @@ -457,36 +466,45 @@ def sw_reset_counter_dict(): if sws.sw_storm_counter_dict[k] >= _high_water_val: msg = "%s remaining in storm state, received %d events (GE to upper threshold: %d)" % ( - k, sws.sw_storm_counter_dict[k], _high_water_val) - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, - tds.CODE_GENERAL, msg) + k, + sws.sw_storm_counter_dict[k], + _high_water_val, + ) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) sws.sw_storm_counter_dict[k] = 0 else: _low_water_val = sws.sw_config_low_water_in_interval_dict[_loc_oid] if sws.sw_storm_counter_dict[k] < _low_water_val: try: msg = "STORM OVER: received %d events (%s) from %s (less than low water threshold: %d)" % ( - sws.sw_storm_counter_dict[k], _loc_oid, _loc_agent, _low_water_val) - ecomp_logger(tds.LOG_TYPE_AUDIT, tds.SEV_WARN, - tds.CODE_GENERAL, msg) + sws.sw_storm_counter_dict[k], + _loc_oid, + _loc_agent, + _low_water_val, + ) + ecomp_logger(tds.LOG_TYPE_AUDIT, tds.SEV_WARN, tds.CODE_GENERAL, msg) del sws.sw_storm_active_dict[k] sws.sw_storm_counter_dict[k] = 0 except Exception as e: - msg = "unable to remove %s from storm active dictionary - TRAPS MAY BE DISCARDED UNINTENTIONALLY! Reason: %s " % ( - k, e) - ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_ERROR, - tds.CODE_GENERAL, msg) + msg = ( + "unable to remove %s from storm active dictionary - TRAPS MAY BE DISCARDED UNINTENTIONALLY! Reason: %s " + % (k, e) + ) + ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_ERROR, tds.CODE_GENERAL, msg) else: msg = "%s remaining in storm state, received %d events (GE to lower threshold: %d)" % ( - k, sws.sw_storm_counter_dict[k], _low_water_val) - ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_INFO, - tds.CODE_GENERAL, msg) + k, + sws.sw_storm_counter_dict[k], + _low_water_val, + ) + ecomp_logger(tds.LOG_TYPE_ERROR, tds.SEV_INFO, tds.CODE_GENERAL, msg) sws.sw_storm_counter_dict[k] = 0 sws.sw_last_stormwatch_dict_analysis = int(time.time()) return True + # # # # # # # # # # # # # # fx: sw_increment_counter # - increment OID and agent trap counters @@ -511,14 +529,11 @@ def sw_increment_counter(_dict_key): try: sws.sw_storm_counter_dict[_dict_key] += 1 - msg = "stormwatch counter for %s now: %d" % ( - _dict_key, sws.sw_storm_counter_dict[_dict_key]) - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, - tds.CODE_GENERAL, msg) + msg = "stormwatch counter for %s now: %d" % (_dict_key, sws.sw_storm_counter_dict[_dict_key]) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) return True except Exception as E: msg = "first trap for %s - init stormwatch counter to 1" % (_dict_key) - ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, - tds.CODE_GENERAL, msg) + ecomp_logger(tds.LOG_TYPE_DEBUG, tds.SEV_INFO, tds.CODE_GENERAL, msg) sws.sw_storm_counter_dict[_dict_key] = 1 return True |