diff options
| author |   Eli Halych <illia.halych@t-mobile.pl> | 2020-09-02 13:10:35 +0000 |
|---|---|---|
| committer | Eli Halych <illia.halych@t-mobile.pl> | 2020-09-07 08:52:24 +0000 |
| commit | 7035aed700a463fd171807526475baf84c1434e7 (patch) | |
| tree | 9d0e7c4c8252f431ac3e12c0f5086c1c8b625f79 /pylog | |
| parent | 314ee85cf7c8a98dd21d5d12e4013b9f742b1012 (diff) | |
onaplogging: Docstrings, refactor, type hinting
Identify and document functionalities. Describe parameters
and their types, exception descriptions and types, extensions,
return types and descriptions. Preserve Python 2.7 and 3.x
compatibility. Add Python 2.7 to Tox testing. Extract code to utility files.
Add properties for readability and maintainability, fix naming conventions.
Deprecate old methods and attributes.
Issue-ID: REQ-420
Signed-off-by: Eli Halych <illia.halych@t-mobile.pl>
Change-Id: I19297e40fad743ec68aa04612ecbb11f61f2abec
Diffstat (limited to 'pylog')
23 files changed, 1432 insertions, 451 deletions
diff --git a/pylog/onaplogging/colorFormatter.py b/pylog/onaplogging/colorFormatter.py index ef713ea..5de0d60 100644 --- a/pylog/onaplogging/colorFormatter.py +++ b/pylog/onaplogging/colorFormatter.py @@ -13,125 +13,207 @@ # limitations under the License. import os -import sys -from logging import Formatter -from .utils import is_above_python_2_7, is_above_python_3_2 +from logging import Formatter, LogRecord +from deprecated import deprecated +from warnings import warn +from typing import Optional, Union, Dict +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 +from onaplogging.utils.styles import ( + ATTRIBUTES, + HIGHLIGHTS, + COLORS, -ATTRIBUTES = { - 'normal': 0, - 'bold': 1, - 'underline': 4, - 'blink': 5, - 'invert': 7, - 'hide': 8, + ATTRIBUTE_TAG, + HIGHLIGHT_TAG, + COLOR_TAG, -} + RESET, + FMT_STR +) -HIGHLIGHTS = { - - 'black': 40, - 'red': 41, - 'green': 42, - 'yellow': 43, - 'blue': 44, - 'purple': 45, - 'cyan': 46, - 'white': 47, -} +class BaseColorFormatter(Formatter): + """Text color formatter class. + + Wraps the logging. Uses Git shell coloring codes. Doesn't support Windows + CMD yet. If `fmt` is not suppied, the `style` is used. Eventually converts + a LogRecord object to "colored" text. + + TODO: + Support for Windows CMD. + Extends: + logging.Formatter + Properties: + style : '%', '{' or '$' formatting. + datefrmt : ISO8601-like (or RFC 3339-like) format. + Args: + fmt : human-readable format. Defaults to None. + datefmt : ISO8601-like (or RFC 3339-like) format. Defaults to None. + colorfmt : Color schemas for logging levels. Defaults to None. + style : '%', '{' or '$' formatting. Defaults to '%'. + Methods: + format : formats a LogRecord record. + _parseColor : selects colors based on a logging levels. + """ + + @property + def style(self): + # type: () -> str + return self.__style # name mangling with __ to avoid accidents + + @property + def colorfmt(self): + # type: () -> str + return self.__colorfmt + + @style.setter + def style(self, value): + # type: (str) -> None + """Assign new style.""" + self.__style = value + + @colorfmt.setter + def colorfmt(self, value): + # type: (str) -> None + """Assign new color format.""" + self.__colorfmt = value + + def __init__(self, + fmt=None, # type: Optional[str] + datefmt=None, # type: Optional[str] + colorfmt=None, # type: Optional[Dict] + style="%"): # type: Optional[str] -COLORS = { + if is_above_python_3_2(): + super(BaseColorFormatter, self). \ + __init__(fmt=fmt, # noqa: E122 + datefmt=datefmt, + style=style) - 'black': 30, - 'red': 31, - 'green': 32, - 'yellow': 33, - 'blue': 34, - 'purple': 35, - 'cyan': 36, - 'white': 37, -} + elif is_above_python_2_7(): + super(BaseColorFormatter, self). \ + __init__(fmt, datefmt) # noqa: E122 -COLOR_TAG = "color" -HIGHLIGHT_TAG = "highlight" -ATTRIBUTE_TAG = "attribute" + else: + Formatter. \ + __init__(self, fmt, datefmt) # noqa: E122 + self.style = style + self.colorfmt = colorfmt -RESET = "\033[0m" -FMT_STR = "\033[%dm%s" + def format(self, record): + """Text formatter. + Connects 2 methods. First it extract a level and a colors + assigned to this level in the BaseColorFormatter class. + Second it applied the colors to the text. -def colored(text, color=None, on_color=None, attrs=None): - # It can't support windows system cmd right now! - # TODO: colered output on windows system cmd - if os.name in ('nt', 'ce'): - return text + Args: + record : an instance of a logged event. + Returns: + str : "colored" text (formatted text). + """ - if isinstance(attrs, str): - attrs = [attrs] + if is_above_python_2_7(): + s = super(BaseColorFormatter, self). \ + format(record) - if os.getenv('ANSI_COLORS_DISABLED', None) is None: - if color is not None and isinstance(color, str): - text = FMT_STR % (COLORS.get(color, 0), text) + else: + s = Formatter. \ + format(self, record) - if on_color is not None and isinstance(on_color, str): - text = FMT_STR % (HIGHLIGHTS.get(on_color, 0), text) + color, highlight, attribute = self._parse_color(record) - if attrs is not None: - for attr in attrs: - text = FMT_STR % (ATTRIBUTES.get(attr, 0), text) + return apply_color(s, color, highlight, attrs=attribute) - # keep origin color for tail spaces - text += RESET - return text + def _parse_color(self, record): + # type: (LogRecord) -> (Optional[str], Optional[str], Optional[str]) + """Color formatter based on the logging level. + This method formats the record according to its level + and a color format set for that level. If the level is + not found, then this method will eventually return None. -class BaseColorFormatter(Formatter): + Args: + record : an instance of a logged event. + Returns: + str : Colors. + str : Hightlight tag. + str : Attribute tag. + """ + if self.colorfmt and \ + isinstance(self.colorfmt, dict): - def __init__(self, fmt=None, datefmt=None, colorfmt=None, style="%"): - if is_above_python_3_2(): - super(BaseColorFormatter, self).__init__( - fmt=fmt, datefmt=datefmt, style=style) - elif is_above_python_2_7(): - super(BaseColorFormatter, self).__init__(fmt, datefmt) - else: - Formatter.__init__(self, fmt, datefmt) + level = record.levelname + colors = self.colorfmt.get(level, None) - self.style = style - self.colorfmt = colorfmt + if colors is not None and \ + isinstance(colors, dict): + return (colors.get(COLOR_TAG, None), # noqa: E201 + colors.get(HIGHLIGHT_TAG, None), + colors.get(ATTRIBUTE_TAG, None)) # noqa: E202 + return None, None, None + @deprecated(reason="Will be removed. Use _parse_color(record) instead.") def _parseColor(self, record): """ - color formatter for instance: - { - "logging-levelname": - { - "color":"<COLORS>", - "highlight":"<HIGHLIGHTS>", - "attribute":"<ATTRIBUTES>", - } - } - :param record: - :return: text color, background color, text attribute + Color based on logging level. + See method _parse_color(record). """ - if self.colorfmt and isinstance(self.colorfmt, dict): + return self._parse_color(record) + + +def apply_color(text, # type: str + color=None, # type: Optional[str] + on_color=None, # type: Optional[str] + attrs=None): # type: Optional[Union[str, list]] + # type: (...) -> str + """Applies color codes to the text. + + Args: + text : text to be "colored" (formatted). + color : Color in human-readable format. Defaults to None. + highlight : Hightlight color in human-readable format. + Previously called "on_color". Defaults to None. + attrs : Colors for attribute(s). Defaults to None. + Returns: + str : "colored" text (formatted text). + """ + warn("`on_color` will be replaced with `highlight`.", DeprecationWarning) + highlight = on_color # replace the parameter and remove - level = record.levelname - colors = self.colorfmt.get(level, None) + if os.name in ('nt', 'ce'): + return text - if colors is not None and isinstance(colors, dict): - return colors.get(COLOR_TAG, None), \ - colors.get(HIGHLIGHT_TAG, None), \ - colors.get(ATTRIBUTE_TAG, None) + if isinstance(attrs, str): + attrs = [attrs] - return None, None, None + ansi_disabled = os.getenv('ANSI_COLORS_DISABLED', None) - def format(self, record): + if ansi_disabled is None: - if sys.version_info > (2, 7): - s = super(BaseColorFormatter, self).format(record) - else: - s = Formatter.format(self, record) - color, on_color, attribute = self._parseColor(record) - return colored(s, color, on_color, attrs=attribute) + if color is not None and \ + isinstance(color, str): + text = FMT_STR % (COLORS.get(color, 0), text) + + if highlight is not None and \ + isinstance(highlight, str): + text = FMT_STR % (HIGHLIGHTS.get(highlight, 0), text) + + if attrs is not None: + for attr in attrs: + text = FMT_STR % (ATTRIBUTES.get(attr, 0), text) + + text += RESET # keep origin color for tail spaces + + return text + + +@deprecated(reason="Will be removed. Call apply_color(...) instead.") +def colored(text, color=None, on_color=None, attrs=None): + """ + Format text with color codes. + See method apply_color(text, color, on_color, attrs). + """ + return apply_color(text, color, on_color, attrs) diff --git a/pylog/onaplogging/logWatchDog.py b/pylog/onaplogging/logWatchDog.py index f93dd12..42e8646 100644 --- a/pylog/onaplogging/logWatchDog.py +++ b/pylog/onaplogging/logWatchDog.py @@ -13,69 +13,131 @@ # limitations under the License. import os -import yaml import traceback + from logging import config +from typing import Dict, Optional, Any +from deprecated import deprecated +from warnings import warn + from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler +from watchdog.events import FileSystemEventHandler, FileSystemEvent + +from onaplogging.utils.tools import yaml_to_dict -__all__ = ['patch_loggingYaml'] +__all__ = ['patch_loggingYaml'] # rename after the deprecated name changed -def _yaml2Dict(filename): +class FileEventHandlers(FileSystemEventHandler): + """Handler of the events in the file system. - with open(filename, 'rt') as f: - return yaml.load(f.read()) + Use it to keep and eye on files in the file system. + Extends: + watchdog.events.FileSystemEventHandler + Properties: + filepath : The path to the file to be monitored. + current_config : Defaults to None. + Args: + filepath : The path to the file to be monitored. + """ -class FileEventHandlers(FileSystemEventHandler): + @property + def filepath(self): + # type: () -> str + return self._filepath - def __init__(self, filepath): + @property + def current_config(self): + # type: () -> str + return self.currentConfig # deprecated, replace with _current_config + + @filepath.setter + def filepath(self, value): + # type: (str) -> str + self._filepath = value + + @current_config.setter + def current_config(self, value): + # type: (Dict) -> Dict + self.currentConfig = value + + def __init__(self, filepath): # type: (str) + warn("Attribute currentConfig will be replaced with property" + "current_config. Use current_config instead.") FileSystemEventHandler.__init__(self) + self.filepath = filepath - self.currentConfig = None + self.current_config = None def on_modified(self, event): + # type: (FileSystemEvent) -> None + """Configuration file actualizer. + + When an event occurs in the file system the hadnler's filepath + is taken to update the configuration file. If the actualization + of the config file fails it will keep the old config file. + + Args: + event : Represents an event on the file system. + Raises: + Exception : If the actualization of the config file fails. + """ try: if event.src_path == self.filepath: - newConfig = _yaml2Dict(self.filepath) - print("reload logging configure file %s" % event.src_path) - config.dictConfig(newConfig) - self.currentConfig = newConfig + + new_config = yaml_to_dict(self.filepath) + print("Reloading logging configuration file %s " + % event.src_path) + + config.dictConfig(new_config) + self.current_config = new_config except Exception: traceback.print_exc() print("Reuse the old configuration to avoid this" "exception terminate program") - if self.currentConfig: - config.dictConfig(self.currentConfig) + + if self.current_config: + config.dictConfig(self.current_config) def _yamlConfig(filepath=None, watchDog=None): + # type: (Optional[str], Optional[Any]) -> None + """YAML configuration file loader. - """ - load logging configureation from yaml file and monitor file status + Use it to monitor a file status in a directory. The watchdog can monitor + a YAML file status looking for modifications. If the watchdog is provided + start observing the directory. The new configuration file is saved as + current for the later reuse. + + Args: + filepath : The path to the file to be monitored. Defaults to None. + watchDog : Monitors a YAML file identifier status. Defaults to None. - :param filepath: logging yaml configure file absolute path - :param watchDog: monitor yaml file identifier status - :return: + Raises: + OSError : If the requested file in the filepath is not a file. + Exception : If watchdog observer setup or YAML coversion fails. """ - if os.path.isfile(filepath) is False: - raise OSError("wrong file") + + is_file = os.path.isfile(filepath) + + if is_file is False: + raise OSError("%s is not a file" % (filepath)) dirpath = os.path.dirname(filepath) event_handler = None try: - dictConfig = _yaml2Dict(filepath) - # The watchdog could monitor yaml file status,if be modified - # will send a notify then we could reload logging configuration + dictConfig = yaml_to_dict(filepath) + # Dev note: Will send a notify then we could reload logging config if watchDog: observer = Observer() event_handler = FileEventHandlers(filepath) - observer.schedule(event_handler=event_handler, path=dirpath, + observer.schedule(event_handler=event_handler, + path=dirpath, recursive=False) observer.setDaemon(True) observer.start() @@ -83,14 +145,23 @@ def _yamlConfig(filepath=None, watchDog=None): config.dictConfig(dictConfig) if event_handler: - # here we keep the correct configuration for reusing event_handler.currentConfig = dictConfig except Exception: traceback.print_exc() -def patch_loggingYaml(): - # The patch to add yam config forlogginf and runtime - # reload logging when modify yaml file +def patch_logging_yaml(): + # type: () -> None + """YAML configuration patch. + + Adds the YAML configuration file loader + to logging.config module during runtime. + """ config.yamlConfig = _yamlConfig + + +@deprecated(reason="Will be removed. Call patch_logging_yaml() instead.") +def patch_loggingYaml(): + """See patch_logging_yaml()""" + patch_logging_yaml() diff --git a/pylog/onaplogging/marker/marker.py b/pylog/onaplogging/marker/marker.py index 5414e21..17a3328 100644 --- a/pylog/onaplogging/marker/marker.py +++ b/pylog/onaplogging/marker/marker.py @@ -14,10 +14,30 @@ import abc +from typing import Iterable, List, Optional, Union, Iterator +from deprecated import deprecated +from warnings import warn +from logging import LogRecord + MARKER_TAG = "marker" class Marker(object): + """Abstract class for defining the marker structure. + + TODO: + after deprecated child methods are removed, rename them here. + Extends: + object + Method list: + getName + addChild + addChilds + removeChild + contains + Raises: + NotImplementedError + """ __metaclass__ = abc.ABCMeta @@ -51,92 +71,235 @@ class Marker(object): class BaseMarker(Marker): + """Basic marker class. + + It is a marker with base functionalities that add sub-level markers and + check if another marker exists as the parent itself or as its child. + + Extends: + Marker + Properties: + name : The name of the marker. + children (list) : The list of all children (sub-level) markers. + Arguments: + name (str) : The name of the marker. + Methods: + getName : Returns the name of the marker. + addChild : Adds a sub-level marker. + addChilds : Adds a list of sub-level markers. + removeChild : Removes a specified sub-level marker. + contains : Checks if a sub-level marker exists. + """ + + @property + def name(self): + # type: () -> str + """Name of the parent marker.""" + return self.__name + + @property + def children(self): + # type: () -> List[Marker] + """Child markers of one parent marker.""" + return self.__childs + + @name.setter + def name(self, value): + # type: (str) -> None + self.__name = value + + @children.setter + def children(self, value): + # type: (List[Marker]) -> None + self.__childs = value + + def __init__(self, name): # type: (str) + """ + Raises: + TypeError : If the `name` parameter is not a string. + ValueError : If the `name` parameter is an empty string. + """ - def __init__(self, name): super(BaseMarker, self).__init__() + if not isinstance(name, str): raise TypeError("not str type") + if name == "": raise ValueError("empty value") + + warn("Attribute `__childs` is replaced by the property `children`." + "Use children instead.", DeprecationWarning) + self.__name = name self.__childs = [] - def getName(self): - return self.__name + def add_child(self, marker): + # type: (Marker) -> None + """Append a marker to child markers. - def __iter__(self): - return iter(self.__childs) + Use this method to describe a different level of logs. For example, + error log would use the ERROR marker. However it's possible to + create a, for instance, TYPE_ERROR to mark type related events. + In this case TYPE_ERROR will be a child of parent ERROR. - def __eq__(self, other): + Args: + marker : marker describing a different log level. + Raises: + TypeError : if the marker object has different type. + """ - if not isinstance(other, Marker): - return False - if id(self) == id(other): - return True + if not isinstance(marker, Marker): + raise TypeError("Bad marker type. \ + Can only add markers of type Marker. \ + Type %s was passed." % type(marker)) - return self.__name == other.getName() + if self == marker: + return - def __hash__(self): - return hash(self.__name) + if marker not in self.children: + self.children.append(marker) - def contains(self, item=None): + def add_children(self, markers): + # type: (Iterable[List]) -> None + """ Append a list of markers to child markers. + + Args: + markers : An iterable object, containing markers. + Raises: + Exception : If `marker` parameter is not iterable. + """ + + try: + iter(markers) + except Exception as e: + raise e + + for marker in markers: + self.children.append(marker) + + def remove_child(self, marker): + # type: (Marker) -> None + """Use this method to remove a marker from the children list. - if isinstance(item, Marker): - if item == self: + Args: + marker : A child marker object. + Raises: + TypeError: if the marker object has different type. + """ + + if not isinstance(marker, Marker): + raise TypeError("Bad marker type. \ + Can only add markers of type Marker. \ + Type %s was passed." % type(marker)) + + if marker in self.children: + self.children.remove(marker) + + def contains(self, item=None): + # type: (Optional[Union[Marker, str]]) -> bool + """ + Use it to check if a marker exists as a parent itself or its chidren. + + Args: + item : A child marker object. Defaults to None. + Returns: + bool : True if the marker exists. + """ + + warn("`item` argument will be replaced with `marker`. " + "Default value None will be removed.", + DeprecationWarning) + marker = item + + if isinstance(marker, Marker): + if marker == self: return True return len(list(filter( - lambda x: x == item, self.__childs))) > 0 + lambda x: x == marker, self.children))) > 0 - elif isinstance(item, str): - if item == self.__name: + elif isinstance(marker, str): + if marker == self.name: return True return len(list(filter( - lambda x: x.__name == item, self.__childs))) > 0 + lambda x: x.name == marker, self.children))) > 0 return False - def addChild(self, item): - if not isinstance(item, Marker): - raise TypeError("can only add (not %s) marker type" - % type(item)) - if self == item: - return - if item not in self.__childs: - self.__childs.append(item) + def __iter__(self): + # type: () -> Iterator[List[Marker]] + return iter(self.__childs) + + def __hash__(self): + # type (): -> int + return hash(self.__name) + def __eq__(self, other): + # type: (Marker) -> bool + if not isinstance(other, Marker): + return False + if id(self) == id(other): + return True + + return self.__name == other.getName() + + @deprecated(reason="Will be removed. Call the `name` property instead.") + def getName(self): + """Class attribute getter.""" + return self.name + + @deprecated(reason="Will be removed. Call add_children(markers) instead.") def addChilds(self, childs): - try: - iter(childs) - except Exception as e: - raise e + """Add a list of sub-level markers. See add_children(markers)""" + self.add_children(childs) - for item in childs: - self.addChild(item) + @deprecated(reason="Will be removed. Call add_child(marker) instead.") + def addChild(self, item): + """Add a sub-level marker. See add_child(marker)""" + self.add_child(item) + @deprecated(reason="Will be removed. Call remove_child(marker) instead.") def removeChild(self, item): - if not isinstance(item, Marker): - raise TypeError("can only add (not %s) marker type" - % type(item)) - if item in self.__childs: - self.__childs.remove(item) + """Remove a sub-level marker. See remove_child(marker)""" + self.remove_child(item) +@deprecated(reason="Will be removed. " + "Call match_marker(record, marker_to_match) instead.") def matchMarkerHelp(record, markerToMatch): - - marker = getattr(record, MARKER_TAG, None) - - if marker is None or markerToMatch is None: + """See match_marker(record, marker_to_match).""" + return match_markers(record, markerToMatch) + + +def match_markers(record, marker_to_match): + # type: (LogRecord, Union[Marker, List]) -> bool + """ + Use this method to match a marker (or a list of markers) with a LogRecord + record. + + Args: + record : a record that may contain a marker. + markerToMatch : a marker or a list of markers. + Raises: + Exception : if match check went wrong. + Returns: + bool : whether the check can be done or the marker is found. + """ + record_marker = getattr(record, MARKER_TAG, None) + + if record_marker is None or \ + marker_to_match is None: return False - if not isinstance(marker, Marker): + if not isinstance(record_marker, Marker): return False try: - if isinstance(markerToMatch, list): + if isinstance(marker_to_match, list): return len(list(filter( - lambda x: marker.contains(x), markerToMatch))) > 0 + lambda x: record_marker.contains(x), marker_to_match))) > 0 - return marker.contains(markerToMatch) + return record_marker.contains(marker_to_match) except Exception as e: raise e diff --git a/pylog/onaplogging/marker/markerFactory.py b/pylog/onaplogging/marker/markerFactory.py index 0705235..a0e9887 100644 --- a/pylog/onaplogging/marker/markerFactory.py +++ b/pylog/onaplogging/marker/markerFactory.py @@ -14,12 +14,32 @@ import abc import threading + +from deprecated import deprecated +from warnings import warn +from typing import Dict, Optional + +from .marker import Marker from .marker import BaseMarker lock = threading.RLock() class IMarkerFactory(object): + """Abstract marker factory for defining structure. + + TODO: + after deprecated child methods are removed, rename them here. + Extends: + object + Method list: + getMarker + deleteMarker + exist + Raises: + NotImplementedError + """ + __metaclass__ = abc.ABCMeta @abc.abstractmethod @@ -36,39 +56,111 @@ class IMarkerFactory(object): class MarkerFactory(IMarkerFactory): + """A factory class maganing every marker. + + It is designed to check the existance, create and remove single markers. + This class follows a singleton pattern - only one instance can be created. + + Extends: + IMarkerFactory + Properties: + marker_map : a map of existing markers. + Attributes: + _instance : a marker factory instance. + Methods: + getMarker : creates a new marker or returns an available one. + deleteMarker : removes a specific marker. + exist : checks if a specific marker exists. + """ _instance = None _marker_map = {} - def __new__(cls, *args, **kwargs): - - if cls._instance is None: - cls._instance = super(MarkerFactory, cls).__new__(cls) + @property + def marker_map(self): + # type: () -> Dict + if not hasattr(self, '_marker_map'): + self._marker_map = {} + return self._marker_map + + def get_marker(self, name=None): + # type: (Optional[str]) -> Marker + """ + Use it to get any marker by its name. If it doesn't exist - it + will create a new marker that will be added to the factory. + Blocks the thread while executing. + + Args: + name : A marker name. Defaults to None. + Raises: + ValueError : If `name` is None. + Returns: + Marker : A found or just newly created marker. + """ + + if name is None: + raise ValueError("Marker name is None. Must have a str value.") - return cls._instance + lock.acquire() - def getMarker(self, marker_name=None): - if marker_name is None: - raise ValueError("not empty") + marker = self.marker_map.get(name, None) - lock.acquire() - marker = self._marker_map.get(marker_name, None) if marker is None: - marker = BaseMarker(name=marker_name) - self._marker_map[marker_name] = marker + marker = BaseMarker(name) + self.marker_map[name] = marker + lock.release() return marker - def deleteMarker(self, marker_name=None): + def delete_marker(self, name=None): + # type: (Optional[str]) -> bool + """ + Args: + name: A marker name. Defaults to None. + Returns: + bool: The status of deletion. + """ + lock.acquire() - if self.exist(marker_name): - del self._marker_map[marker_name] + exists = self.exists(name) + if exists: + del self.marker_map[name] return True lock.release() + return False + def exists(self, name=None): + # type: (Optional[str]) -> bool + """ + Checks whether the search for a marker returns None and returns the + status of the operation. + + Args: + name: marker name. Defaults to None. + Returns: + bool: status of whether the marker was found. + """ + marker = self.marker_map.get(name, None) + return marker is not None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(MarkerFactory, cls).__new__(cls) + + warn("_marker_map attribute will be replaced by marker_map property.", + DeprecationWarning) + return cls._instance + + @deprecated(reason="Will be removed. Call exists(name) instead.") def exist(self, marker_name=None): + return self.exists(marker_name) - return self._marker_map.get( - marker_name, None) is not None + @deprecated(reason="Will be removed. Call get_marker(name) instead.") + def getMarker(self, marker_name=None): + return self.get_marker(marker_name) + + @deprecated(reason="Will be removed. Call delete_marker(name) instead.") + def deleteMarker(self, marker_name=None): + return self.delete_marker(marker_name) diff --git a/pylog/onaplogging/marker/markerFilter.py b/pylog/onaplogging/marker/markerFilter.py index 4f49884..a381d8e 100644 --- a/pylog/onaplogging/marker/markerFilter.py +++ b/pylog/onaplogging/marker/markerFilter.py @@ -12,21 +12,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -from logging import Filter -from .marker import matchMarkerHelp +from logging import Filter, LogRecord +from warnings import warn +from typing import List, Optional, Union + +from onaplogging.utils.system import is_above_python_2_7 + +from .marker import match_markers, Marker class MarkerFilter(Filter): + """Marker filtering. + + Extends: + logging.Filter + Properties: + marker_to_match (Marker/list): a marker of list of markers. + Methods + filter: Filter records by the current filter marker(s). + """ + + @property + def markers_to_match(self): + # type: () -> Union[Marker, List[Marker]] + return self.markersToMatch # TODO renamed - deprecated - def __init__(self, name="", markers=None): - if sys.version_info > (2, 7): + @markers_to_match.setter + def markers_to_match(self, value): + # type: ( Union[Marker, List[Marker]] ) -> None + self.markersToMatch = value + + def __init__(self, + name="", # type: str + markers=None): # type: Optional[Union[Marker, List[Marker]]] + + if is_above_python_2_7(): super(MarkerFilter, self).__init__(name) + else: Filter.__init__(self, name) - self.markerToMatch = markers + warn("markersToMatch attribute will be replaced by a property. " + "Use markers_to_match property instead.", DeprecationWarning) + self.markers_to_match = markers def filter(self, record): - # compare filter's markers with record's marker - return matchMarkerHelp(record, self.markerToMatch) + # type: (LogRecord) -> bool + """Filter by looking for a marker match. + + Args: + record: A record to match with the filter(s). + Returns: + bool: Whether the record matched with the filter(s) + """ + return match_markers(record, self.markers_to_match) diff --git a/pylog/onaplogging/marker/markerHandler.py b/pylog/onaplogging/marker/markerHandler.py index e9ce810..36934a8 100644 --- a/pylog/onaplogging/marker/markerHandler.py +++ b/pylog/onaplogging/marker/markerHandler.py @@ -12,40 +12,118 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys +from logging import LogRecord from logging.handlers import SMTPHandler -from .marker import matchMarkerHelp +from typing import Tuple, List, Optional, Union + +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 + +from .marker import match_markers, Marker class MarkerNotifyHandler(SMTPHandler): + """Handler for email notification. + + Wraps logging.handler.SMTPHandler and extends it by sending only such + notifications which contain certain markers. + + Extends: + SMTPHandler + Property: + markers: A marker or a list of markers. + Args: + mailhost: A (host, port) tuple. + fromaddr: The sender of the email notification. + toaddrs: Email notification recepient(s). + subject: Email subject. + credentials: A (username, password) tuple. + secure: For example (TLS). It is used when the + credentials are supplied. + timout: Default is 5.0 seconds. Python version 3.2+ + markers: A marker or a list of markers. + """ + + @property + def markers(self): + # type: () -> Union[Marker, List[Marker]] + return self._markers + + @markers.setter + def markers(self, value): + # type: ( Union[Marker, List[Marker]] ) - None + self._markers = value + + def __init__(self, + mailhost, # type: Tuple + fromaddr, # type: str + toaddrs, # type: Union[List[str], str] + subject, # type: Tuple + credentials=None, # type: Tuple + secure=None, # type: Optional[Tuple] + timeout=5.0, # type: Optional[float] + markers=None # type: Optional[Union[Marker, List[Marker]]] + ): + + if is_above_python_3_2(): + super(MarkerNotifyHandler, self). \ + __init__( # noqa: E122 + mailhost, + fromaddr, + toaddrs, + subject, + credentials, + secure, + timeout) + + elif is_above_python_2_7(): + super(MarkerNotifyHandler, self). \ + __init__( # noqa: E122 + mailhost, + fromaddr, + toaddrs, + subject, + credentials, + secure) - def __init__(self, mailhost, fromaddr, toaddrs, subject, - credentials=None, secure=None, timeout=5.0, markers=None): - - if sys.version_info > (3, 2): - super(MarkerNotifyHandler, self).__init__( - mailhost, fromaddr, toaddrs, subject, - credentials, secure, timeout) - elif sys.version_info > (2, 7): - super(MarkerNotifyHandler, self).__init__( - mailhost, fromaddr, toaddrs, subject, - credentials, secure) else: SMTPHandler.__init__(self, - mailhost, fromaddr, toaddrs, subject, - credentials, secure) + mailhost, + fromaddr, + toaddrs, + subject, + credentials, + secure) self.markers = markers def handle(self, record): + # type: (LogRecord) -> bool + """ + Handle a LogRecord record. Send an email notification. + """ + return self.send_notification(record) + + def send_notification(self, record): + # type: (LogRecord) -> bool + """Email notification handler. - if self.markers is None: + Matches the record with the specific markers set for email + notifications. Sends an email notification if that marker(s) matched. + + Args: + record (LogRecord): A record that might contain a marker. + Returns: + bool: Whether a record was passed for emission (to be sent). + """ + + if hasattr(self, "markers") and \ + self.markers is None: return False - if matchMarkerHelp(record, self.markers): - if sys.version_info > (2, 7): + if match_markers(record, self.markers): + + if is_above_python_2_7(): return super(MarkerNotifyHandler, self).handle(record) - else: - return SMTPHandler.handle(self, record) + return SMTPHandler.handle(self, record) return False diff --git a/pylog/onaplogging/markerFormatter.py b/pylog/onaplogging/markerFormatter.py index a322e29..d0da695 100644 --- a/pylog/onaplogging/markerFormatter.py +++ b/pylog/onaplogging/markerFormatter.py @@ -12,57 +12,116 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import logging -from .marker import MARKER_TAG -from .marker import Marker +from logging import LogRecord +from typing import Optional + +from onaplogging.utils.styles import MARKER_OPTIONS +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 + +from .marker import Marker, MARKER_TAG from .colorFormatter import BaseColorFormatter class MarkerFormatter(BaseColorFormatter): + """Formats coloring styles based on a marker. - def __init__(self, fmt=None, datefmt=None, colorfmt=None, style='%'): + If `fmt` is not supplied, the `style` is used. - if sys.version_info > (3, 2): - super(MarkerFormatter, self).__init__( - fmt=fmt, datefmt=datefmt, colorfmt=colorfmt, style=style) - elif sys.version_info > (2, 7): - super(MarkerFormatter, self).__init__( - fmt=fmt, datefmt=datefmt, colorfmt=colorfmt) - else: - BaseColorFormatter.__init__(self, fmt, datefmt, colorfmt) + Extends: + BaseColorFormatter + Properties: + marker_tag: a marker to be applied. + temp_fmt : keeps initial format to be reset to after formatting. + Args: + fmt : human-readable format. Defaults to None. + datefmt : ISO8601-like (or RFC 3339-like) format. Defaults to None. + colorfmt : color schemas for logging levels. Defaults to None. + style : '%', '{' or '$' formatting. Defaults to '%'. + Added in Python 3.2. + """ + + @property + def marker_tag(self): + # type: () -> str + return self._marker_tag + + @property + def temp_fmt(self): + # type: () -> str + return self._temp_fmt + + @marker_tag.setter + def marker_tag(self, value): + # type: (str) -> None + self._marker_tag = value - self._marker_tag = "%(marker)s" + @temp_fmt.setter + def temp_fmt(self, value): + # type: (str) -> None + self._temp_fmt = value - if self.style == "{": - self._marker_tag = "{marker}" - elif self.style == "$": - self._marker_tag = "${marker}" + def __init__(self, + fmt=None, # type: Optional[str] + datefmt=None, # type: Optional[str] + colorfmt=None, # type: Optional[dict] + style='%'): # type: Optional[str] - self._tmpFmt = self._fmt + if is_above_python_3_2(): + super(MarkerFormatter, self).\ + __init__(fmt=fmt, # noqa: E122 + datefmt=datefmt, + colorfmt=colorfmt, + style=style) # added in Python 3.2+ + + elif is_above_python_2_7(): + super(MarkerFormatter, self).\ + __init__(fmt=fmt, # noqa: E122 + datefmt=datefmt, + colorfmt=colorfmt) + + else: + BaseColorFormatter.\ + __init__(self, fmt, datefmt, colorfmt) # noqa: E122 + + self.marker_tag = MARKER_OPTIONS[self.style] + self.temp_fmt = self._fmt def format(self, record): + # type: (LogRecord) -> str + """Marker formatter. + + Use it to apply the marker from the LogRecord record to the formatter + string `fmt`. + Args: + record : an instance of a logged event. + Returns: + str : "colored" text (formatted text). + """ try: - if self._fmt.find(self._marker_tag) != -1 \ - and hasattr(record, MARKER_TAG): + + if self._fmt.find(self.marker_tag) != -1 and \ + hasattr(record, MARKER_TAG): marker = getattr(record, MARKER_TAG) if isinstance(marker, Marker): - self._fmt = self._fmt.replace( - self._marker_tag, marker.getName()) - elif self._fmt.find(self._marker_tag) != -1 \ - and not hasattr(record, MARKER_TAG): + self._fmt = self._fmt.replace(self.marker_tag, + marker.name) - self._fmt = self._fmt.replace(self._marker_tag, "") + elif self._fmt.find(self.marker_tag) != -1 and \ + not hasattr(record, MARKER_TAG): + self._fmt = self._fmt.replace(self.marker_tag, "") - if sys.version_info > (3, 2): - self._style = logging._STYLES[self.style][0](self._fmt) + if is_above_python_3_2(): + StylingClass = logging._STYLES[self.style][0] + self.style = StylingClass(self._fmt) - if sys.version_info > (2, 7): + if is_above_python_2_7(): + # includes Python 3.2+ style attribute return super(MarkerFormatter, self).format(record) else: return BaseColorFormatter.format(self, record) finally: - self._fmt = self._tmpFmt + self._fmt = self.temp_fmt diff --git a/pylog/onaplogging/markerLogAdaptor.py b/pylog/onaplogging/markerLogAdaptor.py index e901758..2c1e5df 100644 --- a/pylog/onaplogging/markerLogAdaptor.py +++ b/pylog/onaplogging/markerLogAdaptor.py @@ -12,74 +12,173 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys from logging import LoggerAdapter from threading import RLock from functools import wraps -from .marker import MARKER_TAG -from .marker import Marker +from deprecated import deprecated +from typing import Dict, Callable + +from onaplogging.utils.system import is_above_python_3_2 + +from .marker import Marker, MARKER_TAG from .mdcContext import _getmdcs lock = RLock() -def addMarker(func): +def add_marker(func): + # type: ( Callable[[Marker, str], None] ) -> Callable[[Marker, str], None] + """Marker decorator. + + Requests a blocking acquisition of the thread. Sets the marker + as the logger's marker and delegates a call to the underlying + logger with contextual information. Next it removes the marker + and releases the thread. + + Args: + func : a method supplied with a logging marker. + Raises: + TypeError : the marker type is not `Marker`. + Exception : `extra` doesn't exist or MARKER_TAG is in `extra`. + Returns: + method: decorated method. + """ @wraps(func) def wrapper(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> Callable[[Marker, str], None] + lock.acquire() + if not isinstance(marker, Marker): - raise TypeError("not marker type %s" - % type(marker)) + raise TypeError("Passed a marker of type %s. \ + Should have the type %s." + % type(marker), "Marker") + + if self.extra and \ + MARKER_TAG in self.extra: + raise Exception("Can't add 'marker' in extra - either extra \ + exists or MARKER_TAG is alredy in extra") - if self.extra and MARKER_TAG in self.extra: - raise Exception("cann't add 'marker' in extra") setattr(self.logger, MARKER_TAG, marker) + func(self, marker, msg, *args, **kwargs) + if hasattr(self.logger, MARKER_TAG): delattr(self.logger, MARKER_TAG) + lock.release() + return wrapper +@deprecated(reason="@addMarker is deprecated. Use @add_marker instead.") +def addMarker(func): + """Decorator. See new decorator add_marker(func).""" + add_marker(func) + + class MarkerLogAdaptor(LoggerAdapter): + """Contextual loggin adapter. - def process(self, msg, kwargs): + Specifies contextual information in logging output. Takes a logger and a + dictionary-like object `extra` for providing contextual information. + + An example of the extra contextual information: + extra = {'app_name':'Marker Logging'} + + Extends: + logging.LoggerAdapter + """ - if sys.version_info > (3, 2): + def process(self, msg, kwargs): + # type: (str, Dict) + """Logging call processor. + + Takes a logging message and keyword arguments to provide cotextual + information. + + Args: + msg : Logging information. + kwargs : Contextual information. + Returns: + str : Logging message. + dict : modified (or not) contextual information. + """ + if is_above_python_3_2(): kwargs['extra'] = _getmdcs(self.extra) else: kwargs['extra'] = self.extra return msg, kwargs - @addMarker - def infoMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def info_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with an informational call.""" self.info(msg, *args, **kwargs) - @addMarker - def debugMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def debug_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a debug call.""" self.debug(msg, *args, **kwargs) - @addMarker - def warningMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def warning_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a warning call.""" self.warning(msg, *args, **kwargs) - @addMarker - def errorMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def error_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with an error call.""" self.error(msg, *args, **kwargs) - @addMarker - def exceptionMarker(self, marker, msg, *arg, **kwargs): - self.exception(msg, *arg, **kwargs) + @add_marker + def exception_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with an exceptional call.""" + self.exception(msg, *args, **kwargs) + + @add_marker + def critical_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a critical call.""" + self.critical(msg, *args, **kwargs) + + @add_marker + def log_marker(self, marker, level, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a log call.""" + self.log(marker, level, msg, *args, **kwargs) + + @deprecated(reason="infoMarker(...) is replaced with info_marker(...).") + def infoMarker(self, marker, msg, *args, **kwargs): + self.info_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="debugMarker(...) is replaced with debug_marker(...).") + def debugMarker(self, marker, msg, *args, **kwargs): + self.debug_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="warningMarker(...) replaced, use warning_marker(...).") + def warningMarker(self, marker, msg, *args, **kwargs): + self.warning_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="errorMarker(...) is replaced with error_marker(...).") + def errorMarker(self, marker, msg, *args, **kwargs): + self.error_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="exceptionMarker(...) replaced," + " use exception_marker(...).") + def exceptionMarker(self, marker, msg, *args, **kwargs): + self.exception_marker(marker, msg, *args, **kwargs) - @addMarker - def criticalMarker(self, marker, msg, *arg, **kwargs): - self.critical(msg, *arg, **kwargs) + @deprecated(reason="criticalMarker(...) is replaced, " + "use critical_marker(...).") + def criticalMarker(self, marker, msg, *args, **kwargs): + self.critical_marker(marker, msg, *args, **kwargs) - @addMarker - def logMarker(self, marker, level, msg, *arg, **kwargs): - self.log(level, msg, *arg, **kwargs) + @deprecated(reason="logMarker(...) is replaced with info_marker(...).") + def logMarker(self, marker, level, msg, *args, **kwargs): + self.log_marker(marker, level, msg, *args, **kwargs) diff --git a/pylog/onaplogging/mdcContext.py b/pylog/onaplogging/mdcContext.py index ecdc2d9..c1852b3 100644 --- a/pylog/onaplogging/mdcContext.py +++ b/pylog/onaplogging/mdcContext.py @@ -19,9 +19,16 @@ import os import traceback import sys import functools -from .marker import Marker -from .marker import MARKER_TAG +from deprecated import deprecated +from typing import Dict, Optional, Any, Callable, List, Tuple +from logging import LogRecord + +from onaplogging.utils.system import is_above_python_3_2 + +from .marker import Marker, MARKER_TAG + +# TODO change to patch_logging_mdc after deprecated method is removed __all__ = ['patch_loggingMDC', 'MDC'] _replace_func_name = ['info', 'critical', 'fatal', 'debug', @@ -29,123 +36,186 @@ _replace_func_name = ['info', 'critical', 'fatal', 'debug', 'handle', 'findCaller'] -class MDCContext(threading.local): +def fetchkeys(func): # type: Callable[[str, List, Dict], None] + # type: (...) -> Callable[[str, List, Dict], None] + """MDC decorator. + + Fetchs contextual information from a logging call. + Wraps by adding MDC to the `extra` field. Executes + the call with the updated contextual information. """ - A Thread local instance to storage mdc values + + @functools.wraps(func) + def replace(*args, **kwargs): + # type: () -> None + kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None)) + func(*args, **kwargs) + + return replace + + +class MDCContext(threading.local): + """A Thread local instance that stores MDC values. + + Is initializ with an empty dictionary. Manages that + dictionary to created Mapped Diagnostic Context. + + Extends: + threading.local + Property: + local_dict : a placeholder for MDC keys and values. """ - def __init__(self): + @property + def local_dict(self): + # type: () -> Dict + return self._local_dict + + @local_dict.setter + def local_dict(self, value): + # type: (Dict) -> None + self._local_dict = value + + def __init__(self): super(MDCContext, self).__init__() - self._localDict = {} + self.local_dict = {} def get(self, key): - - return self._localDict.get(key, None) + # type: (str) -> Any + """Retrieve a value by key.""" + return self.local_dict.get(key, None) def put(self, key, value): - - self._localDict[key] = value + # type: (str, Any) -> None + """Insert or update a value by key.""" + self.local_dict[key] = value def remove(self, key): - - if key in self._localDict: - del self._localDict[key] + # type: (str) -> None + """Remove a value by key, if exists.""" + if key in self.local_dict: + del self.local_dict[key] def clear(self): + # type: () -> None + """Empty the MDC dictionary.""" + self.local_dict.clear() - self._localDict.clear() - + @deprecated(reason="Use local_mdc property instead.") def result(self): + """Getter for the MDC dictionary.""" + return self.local_dict - return self._localDict + def empty(self): + # type: () -> bool + """Checks whether the local dictionary is empty.""" + return self.local_dict == {} or \ + self.local_dict is None + @deprecated(reason="Will be replaced. Use empty() instead.") def isEmpty(self): - - return self._localDict == {} or self._localDict is None + """See empty().""" + return self.empty() MDC = MDCContext() -def fetchkeys(func): - - @functools.wraps(func) - def replace(*args, **kwargs): - kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None)) - func(*args, **kwargs) - return replace - - def _getmdcs(extra=None): + # type: (Optional[Dict]) -> Dict """ - Put mdc dict in logging record extra filed with key 'mdc' - :param extra: dict - :return: mdc dict + Puts an MDC dict in the `extra` field with key 'mdc'. This provides + the contextual information with MDC. + + Args: + extra : Contextual information. Defaults to None. + Raises: + KeyError : a key from extra is attempted to be overwritten. + Returns: + dict : contextual information named `extra` with MDC. """ - if MDC.isEmpty(): + if MDC.empty(): return extra - mdc = MDC.result() + mdc = MDC.local_dict if extra is not None: for key in extra: - # make sure extra key dosen't override mdckey - if key in mdc or key == 'mdc': + if key in mdc or \ + key == 'mdc': raise KeyError("Attempt to overwrite %r in MDC" % key) else: extra = {} extra['mdc'] = mdc - del mdc + return extra @fetchkeys def info(self, msg, *args, **kwargs): - + # type: (str) -> None + """If INFO enabled, deletage an info call with MDC.""" if self.isEnabledFor(logging.INFO): self._log(logging.INFO, msg, args, **kwargs) @fetchkeys def debug(self, msg, *args, **kwargs): + # type: (str) -> None + """If DEBUG enabled, deletage a debug call with MDC.""" if self.isEnabledFor(logging.DEBUG): self._log(logging.DEBUG, msg, args, **kwargs) @fetchkeys def warning(self, msg, *args, **kwargs): + # type: (str) -> None + """If WARNING enabled, deletage a warning call with MDC.""" if self.isEnabledFor(logging.WARNING): self._log(logging.WARNING, msg, args, **kwargs) @fetchkeys def exception(self, msg, *args, **kwargs): - + # type: (str) -> None + """Deletage an exception call and set exc_info code to 1.""" kwargs['exc_info'] = 1 self.error(msg, *args, **kwargs) @fetchkeys def critical(self, msg, *args, **kwargs): - + # type: (str) -> None + """If CRITICAL enabled, deletage a critical call with MDC.""" if self.isEnabledFor(logging.CRITICAL): self._log(logging.CRITICAL, msg, args, **kwargs) @fetchkeys def error(self, msg, *args, **kwargs): + # type: (str) -> None + """If ERROR enabled, deletage an error call with MDC.""" if self.isEnabledFor(logging.ERROR): self._log(logging.ERROR, msg, args, **kwargs) @fetchkeys def log(self, level, msg, *args, **kwargs): + # type: (int, str) -> None + """ + If a specific logging level enabled and the code is represented + as an integer value, delegate the call to the underlying logger. + + Raises: + TypeError: if the logging level code is not an integer. + """ if not isinstance(level, int): if logging.raiseExceptions: - raise TypeError("level must be an integer") + raise TypeError("Logging level code must be an integer." + "Got %s instead." % type(level)) else: return @@ -154,55 +224,80 @@ def log(self, level, msg, *args, **kwargs): def handle(self, record): - + # type: (LogRecord) -> None cmarker = getattr(self, MARKER_TAG, None) if isinstance(cmarker, Marker): setattr(record, MARKER_TAG, cmarker) - if (not self.disabled) and self.filter(record): + if not self.disabled and \ + self.filter(record): self.callHandlers(record) def findCaller(self, stack_info=False): + # type: (bool) -> Tuple + """ + Find the stack frame of the caller so that we can note the source file + name, line number and function name. Enhances the logging.findCaller(). + """ + + frame = logging.currentframe() - f = logging.currentframe() - if f is not None: - f = f.f_back + if frame is not None: + frame = frame.f_back rv = "(unkown file)", 0, "(unknow function)" - while hasattr(f, "f_code"): - co = f.f_code + + while hasattr(frame, "f_code"): + co = frame.f_code filename = os.path.normcase(co.co_filename) # jump through local 'replace' func frame - if filename == logging._srcfile or co.co_name == "replace": - f = f.f_back + if filename == logging._srcfile or \ + co.co_name == "replace": + + frame = frame.f_back continue - if sys.version_info > (3, 2): + + if is_above_python_3_2(): + sinfo = None if stack_info: + sio = io.StringIO() sio.write("Stack (most recent call last):\n") - traceback.print_stack(f, file=sio) + traceback.print_stack(frame, file=sio) sinfo = sio.getvalue() + if sinfo[-1] == '\n': sinfo = sinfo[:-1] + sio.close() - rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) + rv = (co.co_filename, frame.f_lineno, co.co_name, sinfo) + else: - rv = (co.co_filename, f.f_lineno, co.co_name) + rv = (co.co_filename, frame.f_lineno, co.co_name) break return rv -def patch_loggingMDC(): - """ - The patch to add MDC ability in logging Record instance at runtime +def patch_logging_mdc(): + # type: () -> None + """MDC patch. + + Sets MDC in a logging record instance at runtime. """ localModule = sys.modules[__name__] + for attr in dir(logging.Logger): if attr in _replace_func_name: newfunc = getattr(localModule, attr, None) if newfunc: setattr(logging.Logger, attr, newfunc) + + +@deprecated(reason="Will be removed. Call patch_logging_mdc() instead.") +def patch_loggingMDC(): + """See patch_logging_ymdc().""" + patch_logging_mdc() diff --git a/pylog/onaplogging/mdcformatter.py b/pylog/onaplogging/mdcformatter.py index 4cacbe8..442f0ad 100644 --- a/pylog/onaplogging/mdcformatter.py +++ b/pylog/onaplogging/mdcformatter.py @@ -13,26 +13,60 @@ # limitations under the License. import logging +from logging import LogRecord +from typing import Mapping, List, Dict, Callable +from deprecated import deprecated + +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 +from onaplogging.utils.styles import MDC_OPTIONS + from .markerFormatter import MarkerFormatter -from .utils import is_above_python_2_7, is_above_python_3_2 class MDCFormatter(MarkerFormatter): - """ - A custom MDC formatter to prepare Mapped Diagnostic Context - to enrich log message. + """A custom MDC formatter. + + Prepares Mapped Diagnostic Context to enrich log message. If `fmt` is not + supplied, the `style` is used. + + Extends: + MarkerFormatter + Args: + fmt : Built-in format string containing standard Python + %-style mapping keys in human-readable format. + mdcFmt : MDC format with '{}'-style mapping keys. + datefmt : Date format. + colorfmt : colored output with an ANSI terminal escape code. + style : style mapping keys in Python 3.x. """ - def __init__(self, fmt=None, mdcfmt=None, - datefmt=None, colorfmt=None, style="%"): - """ - :param fmt: build-in format string contains standard - Python %-style mapping keys - :param mdcFmt: mdc format with '{}'-style mapping keys - :param datefmt: Date format to use - :param colorfmt: colored output with ANSI escape code on terminal - :param style: style mapping keys in python3 - """ + @property + def mdc_tag(self): + # type: () -> str + return self._mdc_tag + + @property + def mdcfmt(self): + # type: () -> str + return self._mdcFmt + + @mdc_tag.setter + def mdc_tag(self, value): + # type: (str) -> str + self._mdc_tag = value + + @mdcfmt.setter + def mdcfmt(self, value): + # type: (str) -> str + self._mdc_tag = value + + def __init__(self, + fmt=None, # type: str + mdcfmt=None, # type: str + datefmt=None, # type: str + colorfmt=None, # type: str + style="%"): # type: str + if is_above_python_3_2(): super(MDCFormatter, self).__init__(fmt=fmt, datefmt=datefmt, @@ -43,114 +77,149 @@ class MDCFormatter(MarkerFormatter): datefmt=datefmt, colorfmt=colorfmt) else: - MarkerFormatter.__init__(self, fmt, datefmt, colorfmt) + MarkerFormatter.\ + __init__(self, fmt, datefmt, colorfmt) # noqa: E122 - self._mdc_tag = "%(mdc)s" - if self.style == "{": - self._mdc_tag = "{mdc}" - elif self.style == "$": - self._mdc_tag = "${mdc}" - - if mdcfmt: - self._mdcFmt = mdcfmt - else: - self._mdcFmt = '{reqeustID}' + self._mdc_tag = MDC_OPTIONS[self.style] + self._mdcFmt = mdcfmt if mdcfmt else '{reqeustID}' - def _mdcfmtKey(self): + def format(self, record): + # type: (LogRecord) -> str """ - maximum barce match algorithm to find the mdc key - :return: key in brace and key not in brace,such as ({key}, key) + Find MDCs in a log record's extra field. If the key from mdcFmt + doesn't contain MDC, the values will be empty. + + For example: + The MDC dict in a logging record is {'key1':'value1','key2':'value2'}. + The mdcFmt is '{key1} {key3}'. + The output of MDC message is 'key1=value1 key3='. + + Args: + record : an instance of a logged event. + Returns: + str : "colored" text (formatted text). + """ + + mdc_index = self._fmt.find(self._mdc_tag) + if mdc_index == -1: + return self._parent_format(record) + + mdc_format_keys, mdc_format_words = self._mdc_format_key() + + if mdc_format_words is None: + self._fmt = self._replace_mdc_tag_str("") + self._apply_styling() + + return self._parent_format(record) + + res = self._apply_mdc(record, mdc_format_words) + + try: + mdc_string = self._replaceStr(keys=mdc_format_keys).format(**res) + self._fmt = self._replace_mdc_tag_str(mdc_string) + self._apply_styling() + + return self._parent_format(record) + + except KeyError as e: + # is there a need for print? + print("The mdc key %s format is wrong" % str(e)) + + except Exception: + raise + + def _mdc_format_key(self): + # type: () -> (List, Mapping[str, str]) + """Maximum (balanced) parantehses matching algorithm for MDC keys. + + Extracts and strips keys and words from a MDC format string. Use this + method to find the MDC key. + + Returns: + list : list of keys. + map object : keys with and without brace, such as ({key}, key). """ left = '{' right = '}' target = self._mdcFmt - st = [] + stack = [] keys = [] + for index, v in enumerate(target): if v == left: - st.append(index) + stack.append(index) elif v == right: - if len(st) == 0: + if len(stack) == 0: continue - elif len(st) == 1: - start = st.pop() + elif len(stack) == 1: + start = stack.pop() end = index keys.append(target[start:end + 1]) - elif len(st) > 0: - st.pop() + elif len(stack) > 0: + stack.pop() keys = list(filter(lambda x: x[1:-1].strip('\n \t ') != "", keys)) words = None + if keys: words = map(lambda x: x[1:-1], keys) return keys, words - def _replaceStr(self, keys): - - fmt = self._mdcFmt - for i in keys: - fmt = fmt.replace(i, i[1:-1] + "=" + i) - - return fmt - - def format(self, record): + def _replace_string(self, keys): + # type: (List[str]) -> str """ - Find mdcs in log record extra field, if key form mdcFmt dosen't - contains mdcs, the values will be empty. - :param record: the logging record instance - :return: string - for example: - the mdcs dict in logging record is - {'key1':'value1','key2':'value2'} - the mdcFmt is" '{key1} {key3}' - the output of mdc message: 'key1=value1 key3=' - + Removes the first and last characters from each key and assigns not + stripped keys. """ - mdcIndex = self._fmt.find(self._mdc_tag) - if mdcIndex == -1: - if is_above_python_2_7(): - return super(MDCFormatter, self).format(record) - else: - return MarkerFormatter.format(self, record) - - mdcFmtkeys, mdcFmtWords = self._mdcfmtKey() - - if mdcFmtWords is None: - self._fmt = self._fmt.replace(self._mdc_tag, "") - if is_above_python_3_2(): - self._style = logging._STYLES[self.style][0](self._fmt) + fmt = self._mdcFmt + for key in keys: + fmt = fmt.replace(key, key[1:-1] + "=" + key) + return fmt - if is_above_python_2_7(): - return super(MDCFormatter, self).format(record) - else: - return MarkerFormatter.format(self, record) + def _parent_format(self, record): + # type: (LogRecord) -> str + """Call super class's format based on Python version.""" + if is_above_python_2_7(): + return super(MDCFormatter, self).format(record) + else: + return MarkerFormatter.format(self, record) + def _apply_mdc(self, record, mdc_format_words): + # type: (LogRecord, Mapping[Callable[[str], str], List]) -> Dict + """Apply MDC pamming to the LogRecord record.""" mdc = record.__dict__.get('mdc', None) res = {} - for i in mdcFmtWords: + + for i in mdc_format_words: if mdc and i in mdc: res[i] = mdc[i] else: res[i] = "" - del mdc - try: - mdcstr = self._replaceStr(keys=mdcFmtkeys).format(**res) - self._fmt = self._fmt.replace(self._mdc_tag, mdcstr) + return res - if is_above_python_3_2(): - self._style = logging._STYLES[self.style][0](self._fmt) + def _apply_styling(self): + # type: () -> None + """Apply styling to the formatter if using Python 3.2+""" + if is_above_python_3_2(): + StylingClass = logging._STYLES[self.style][0](self._fmt) + self._style = StylingClass - if is_above_python_2_7(): - return super(MDCFormatter, self).format(record) - else: - return MarkerFormatter.format(self, record) + def _replace_mdc_tag_str(self, replacement): + # type: (str) -> str + """Replace MDC tag in the format string.""" + return self._fmt.replace(self._mdc_tag, replacement) - except KeyError as e: - print("The mdc key %s format is wrong" % str(e)) - except Exception: - raise + @deprecated(reason="Will be replaced. Use _mdc_format_key() instead.") + def _mdcfmtKey(self): + """See _mdc_format_key().""" + return self._mdc_format_key() + + @deprecated(reason="Will be replaced. Use _replace_string(keys) instead.") + def _replaceStr(self, keys): + """See _replace_string(keys).""" + return self._replace_string(keys) diff --git a/pylog/onaplogging/monkey.py b/pylog/onaplogging/monkey.py index f8bf992..33c3ced 100644 --- a/pylog/onaplogging/monkey.py +++ b/pylog/onaplogging/monkey.py @@ -12,17 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .mdcContext import patch_loggingMDC -from .logWatchDog import patch_loggingYaml +from typing import Optional + +from .mdcContext import patch_logging_mdc +from .logWatchDog import patch_logging_yaml __all__ = ["patch_all"] def patch_all(mdc=True, yaml=True): + # type: ( Optional[bool], Optional[bool] ) -> None + """ + Patches both MDC contextual information and YAML configuration file to the + logger by default. To exclude any or both set `mdc` and/or `yaml` + parameters to False. + + Args: + mdc (bool, optional): Defaults to True. + yaml (bool, optional): Defaults to True. + """ if mdc is True: - patch_loggingMDC() + patch_logging_mdc() if yaml is True: - patch_loggingYaml() + patch_logging_yaml() diff --git a/pylog/onaplogging/utils/__init__.py b/pylog/onaplogging/utils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pylog/onaplogging/utils/__init__.py diff --git a/pylog/onaplogging/utils/styles.py b/pylog/onaplogging/utils/styles.py new file mode 100644 index 0000000..e445aee --- /dev/null +++ b/pylog/onaplogging/utils/styles.py @@ -0,0 +1,84 @@ +# Copyright (c) 2020 Deutsche Telekom. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +"""These are ANSI shell coloring codes used to format strings. + +[ begins the color definition. \033 starts the escape sequence. +[\0330m is the default color of the shell that closes the escape sequence. + +`FMT_STR` takes the color as its first parameter (int). As the second +parameter its takes the text (str). + +TL;DR + Examples on ANSI colors, attributes, backgrounds and foregrounds: + https://stackoverflow.com/a/28938235/7619961 +""" + +COLOR_TAG = "color" +HIGHLIGHT_TAG = "highlight" +ATTRIBUTE_TAG = "attribute" + +RESET = "\033[0m" +FMT_STR = "\033[%dm%s" + +ATTRIBUTES = { + + 'normal': 0, + 'bold': 1, + 'underline': 4, + 'blink': 5, + 'invert': 7, + 'hide': 8, + +} + +HIGHLIGHTS = { + + 'black': 40, + 'red': 41, + 'green': 42, + 'yellow': 43, + 'blue': 44, + 'purple': 45, + 'cyan': 46, + 'white': 47, + +} + +COLORS = { + + 'black': 30, + 'red': 31, + 'green': 32, + 'yellow': 33, + 'blue': 34, + 'purple': 35, + 'cyan': 36, + 'white': 37, + +} + +""" +MDC and MARKER options are used only with Python starting 3.2 due to an update +in the logging module. This allows the use of %-formatting, :meth:`str.format` +(``{}``) formatting or :class:`string.Template` in the format string. +""" + +MARKER_OPTIONS = { + "%": "%(marker)s", + "{": "{marker}", + "$": "${marker}" +} + +MDC_OPTIONS = { + "%": "%(mdc)s", + "{": "{mdc}", + "$": "${mdc}" +} diff --git a/pylog/onaplogging/utils.py b/pylog/onaplogging/utils/system.py index 5c96b2d..cfe49a1 100644 --- a/pylog/onaplogging/utils.py +++ b/pylog/onaplogging/utils/system.py @@ -11,21 +11,21 @@ import sys -def is_above_python_3_2(): # type: () -> bool +def is_above_python_3_2(): + # type: () -> bool """Check if code is running at least on Python 3.2 version. Returns: bool: True if it's at least 3.2 version, False otherwise - """ return sys.version_info >= (3, 2, 0, "final", 0) -def is_above_python_2_7(): # type: () -> bool +def is_above_python_2_7(): + # type: () -> bool """Check if code is running at least on Python 2.7 version. Returns: bool: True if it's at least 2.7 version, False otherwise - """ return sys.version_info >= (2, 7, 0, "final", 0) diff --git a/pylog/onaplogging/utils/tools.py b/pylog/onaplogging/utils/tools.py new file mode 100644 index 0000000..0cb0129 --- /dev/null +++ b/pylog/onaplogging/utils/tools.py @@ -0,0 +1,32 @@ +# Copyright (c) 2020 Deutsche Telekom. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import yaml + +from deprecated import deprecated + + +def yaml_to_dict(filepath): + # type: (str) -> dict + """YAML to Python dict converter. + + Args: + filepath : The filepath to a YAML file. + Returns: + dict : Python dictionary object. + """ + with open(filepath, 'rt') as f: + return yaml.load(f.read()) + + +@deprecated(reason="Will be removed. Call yaml_to_dict(filepath) instead.") +def _yaml2Dict(filename): + """YAML to dict. See yaml_to_dict(filepath).""" + return yaml_to_dict(filename) diff --git a/pylog/requirements.txt b/pylog/requirements.txt index 16afb32..be12b1b 100644 --- a/pylog/requirements.txt +++ b/pylog/requirements.txt @@ -1,2 +1,4 @@ PyYAML watchdog +deprecated +typing diff --git a/pylog/test_requirements.txt b/pylog/test_requirements.txt index d32736b..a5c4199 100644 --- a/pylog/test_requirements.txt +++ b/pylog/test_requirements.txt @@ -1,3 +1,5 @@ -r requirements.txt pytest -mock
\ No newline at end of file +mock +deprecated +typing
\ No newline at end of file diff --git a/pylog/tests/test_color_formatter.py b/pylog/tests/test_color_formatter.py index 9a9ae5a..56bf9c2 100644 --- a/pylog/tests/test_color_formatter.py +++ b/pylog/tests/test_color_formatter.py @@ -27,7 +27,7 @@ from onaplogging.colorFormatter import ( FMT_STR, RESET, ) -from onaplogging.utils import is_above_python_3_2 +from onaplogging.utils.system import is_above_python_3_2 class TestColorFormatter(unittest.TestCase): diff --git a/pylog/tests/test_log_watchdog.py b/pylog/tests/test_log_watchdog.py index 5f43138..e1b9fea 100644 --- a/pylog/tests/test_log_watchdog.py +++ b/pylog/tests/test_log_watchdog.py @@ -22,7 +22,8 @@ if sys.version_info[0] >= 3: import pytest import yaml -from onaplogging.logWatchDog import FileEventHandlers, _yaml2Dict, _yamlConfig +from onaplogging.logWatchDog import FileEventHandlers, _yamlConfig +from onaplogging.utils.tools import yaml_to_dict, _yaml2Dict TestEvent = namedtuple("TestEvent", ["src_path"]) diff --git a/pylog/tests/test_marker_formatter.py b/pylog/tests/test_marker_formatter.py index bda7f24..c7ae6b1 100644 --- a/pylog/tests/test_marker_formatter.py +++ b/pylog/tests/test_marker_formatter.py @@ -30,16 +30,16 @@ class TestMarkerFormatter(unittest.TestCase): def test_marker_formatter_init(self): marker_formatter = MarkerFormatter() self.assertEqual(marker_formatter.style, "%") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") if sys.version_info[0] >= 3: marker_formatter = MarkerFormatter(style="{") self.assertEqual(marker_formatter.style, "{") - self.assertEqual(marker_formatter._marker_tag, "{marker}") + self.assertEqual(marker_formatter.marker_tag, "{marker}") marker_formatter = MarkerFormatter(style="$") self.assertEqual(marker_formatter.style, "$") - self.assertEqual(marker_formatter._marker_tag, "${marker}") + self.assertEqual(marker_formatter.marker_tag, "${marker}") with pytest.raises(ValueError): MarkerFormatter(style="*") @@ -50,18 +50,18 @@ class TestMarkerFormatter(unittest.TestCase): with patch("onaplogging.markerFormatter.BaseColorFormatter.format") as mock_format: marker_formatter = MarkerFormatter() self.assertEqual(marker_formatter._fmt, "%(message)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") marker_formatter.format(record) mock_format.assert_called_once() self.assertEqual(marker_formatter._fmt, "%(message)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") with patch("onaplogging.markerFormatter.BaseColorFormatter.format") as mock_format: marker_formatter = MarkerFormatter(fmt="%(message)s %(marker)s") self.assertEqual(marker_formatter._fmt, "%(message)s %(marker)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") marker_formatter.format(record) mock_format.assert_called_once() self.assertEqual(marker_formatter._fmt, "%(message)s %(marker)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") - self.assertEqual(marker_formatter._tmpFmt, "%(message)s %(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.temp_fmt, "%(message)s %(marker)s") diff --git a/pylog/tests/test_monkey.py b/pylog/tests/test_monkey.py index 4f71fe2..9b64b62 100644 --- a/pylog/tests/test_monkey.py +++ b/pylog/tests/test_monkey.py @@ -16,32 +16,32 @@ if sys.version_info[0] < 3: if sys.version_info[0] >= 3: from unittest.mock import patch -from onaplogging.monkey import patch_all, patch_loggingMDC, patch_loggingYaml +from onaplogging.monkey import patch_all, patch_logging_yaml, patch_logging_mdc class TestMonkey(unittest.TestCase): def test_patch_all(self): - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all() mock_mdc.assert_called_once() mock_yaml.assert_called_once() - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all(mdc=False) mock_mdc.assert_not_called() mock_yaml.assert_called_once() - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all(yaml=False) mock_mdc.assert_called_once() mock_yaml.assert_not_called() - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all(mdc=False, yaml=False) mock_mdc.assert_not_called() mock_yaml.assert_not_called() diff --git a/pylog/tests/test_utils.py b/pylog/tests/test_utils.py index 5a64aab..a118361 100644 --- a/pylog/tests/test_utils.py +++ b/pylog/tests/test_utils.py @@ -16,30 +16,30 @@ if sys.version_info[0] < 3: if sys.version_info[0] >= 3: from unittest.mock import patch, MagicMock -from onaplogging.utils import is_above_python_2_7, is_above_python_3_2 +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 class TestUtils(unittest.TestCase): def test_is_above_python_3_2(self): - with patch("onaplogging.utils.sys.version_info", (3, 4, 7)): + with patch("onaplogging.utils.system.sys.version_info", (3, 4, 7)): assert is_above_python_3_2() is True - with patch("onaplogging.utils.sys.version_info", (2, 7, 5)): + with patch("onaplogging.utils.system.sys.version_info", (2, 7, 5)): assert is_above_python_3_2() is False - with patch("onaplogging.utils.sys.version_info", (3, 2, 0, "final", 0)): + with patch("onaplogging.utils.system.sys.version_info", (3, 2, 0, "final", 0)): assert is_above_python_3_2() is True def test_is_above_python_2_7(self): - with patch("onaplogging.utils.sys.version_info", (3, 4, 7)): + with patch("onaplogging.utils.system.sys.version_info", (3, 4, 7)): assert is_above_python_2_7() is True - with patch("onaplogging.utils.sys.version_info", (2, 7, 5)): + with patch("onaplogging.utils.system.sys.version_info", (2, 7, 5)): assert is_above_python_2_7() is True - with patch("onaplogging.utils.sys.version_info", (2, 5, 6)): + with patch("onaplogging.utils.system.sys.version_info", (2, 5, 6)): assert is_above_python_2_7() is False - with patch("onaplogging.utils.sys.version_info", (2, 7, 0, "final", 0)): + with patch("onaplogging.utils.system.sys.version_info", (2, 7, 0, "final", 0)): assert is_above_python_2_7() is True diff --git a/pylog/tox.ini b/pylog/tox.ini index b5f2dd3..355385c 100644 --- a/pylog/tox.ini +++ b/pylog/tox.ini @@ -1,5 +1,6 @@ [tox] envlist = + py27 py36 py37 py38 @@ -18,6 +19,9 @@ deps = -r{toxinidir}/test_requirements.txt setenv = PYTHONPATH={toxinidir}/ commands = pytest +[flake8] +ignore = E271, E125, E128, E127 + [testenv:pep8] basepython = python3 deps=flake8 |