diff options
24 files changed, 1435 insertions, 452 deletions
@@ -18,10 +18,12 @@ build/ .tox/ __pycache__/ *.pyc +.pytest_cache/ npm-debug.log node_modules/ node_install/ test_output/ test-output/ -dist/
\ No newline at end of file +dist/ +venv/
\ No newline at end of file diff --git a/pylog/onaplogging/colorFormatter.py b/pylog/onaplogging/colorFormatter.py index ef713ea..5de0d60 100644 --- a/pylog/onaplogging/colorFormatter.py +++ b/pylog/onaplogging/colorFormatter.py @@ -13,125 +13,207 @@ # limitations under the License. import os -import sys -from logging import Formatter -from .utils import is_above_python_2_7, is_above_python_3_2 +from logging import Formatter, LogRecord +from deprecated import deprecated +from warnings import warn +from typing import Optional, Union, Dict +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 +from onaplogging.utils.styles import ( + ATTRIBUTES, + HIGHLIGHTS, + COLORS, -ATTRIBUTES = { - 'normal': 0, - 'bold': 1, - 'underline': 4, - 'blink': 5, - 'invert': 7, - 'hide': 8, + ATTRIBUTE_TAG, + HIGHLIGHT_TAG, + COLOR_TAG, -} + RESET, + FMT_STR +) -HIGHLIGHTS = { - - 'black': 40, - 'red': 41, - 'green': 42, - 'yellow': 43, - 'blue': 44, - 'purple': 45, - 'cyan': 46, - 'white': 47, -} +class BaseColorFormatter(Formatter): + """Text color formatter class. + + Wraps the logging. Uses Git shell coloring codes. Doesn't support Windows + CMD yet. If `fmt` is not suppied, the `style` is used. Eventually converts + a LogRecord object to "colored" text. + + TODO: + Support for Windows CMD. + Extends: + logging.Formatter + Properties: + style : '%', '{' or '$' formatting. + datefrmt : ISO8601-like (or RFC 3339-like) format. + Args: + fmt : human-readable format. Defaults to None. + datefmt : ISO8601-like (or RFC 3339-like) format. Defaults to None. + colorfmt : Color schemas for logging levels. Defaults to None. + style : '%', '{' or '$' formatting. Defaults to '%'. + Methods: + format : formats a LogRecord record. + _parseColor : selects colors based on a logging levels. + """ + + @property + def style(self): + # type: () -> str + return self.__style # name mangling with __ to avoid accidents + + @property + def colorfmt(self): + # type: () -> str + return self.__colorfmt + + @style.setter + def style(self, value): + # type: (str) -> None + """Assign new style.""" + self.__style = value + + @colorfmt.setter + def colorfmt(self, value): + # type: (str) -> None + """Assign new color format.""" + self.__colorfmt = value + + def __init__(self, + fmt=None, # type: Optional[str] + datefmt=None, # type: Optional[str] + colorfmt=None, # type: Optional[Dict] + style="%"): # type: Optional[str] -COLORS = { + if is_above_python_3_2(): + super(BaseColorFormatter, self). \ + __init__(fmt=fmt, # noqa: E122 + datefmt=datefmt, + style=style) - 'black': 30, - 'red': 31, - 'green': 32, - 'yellow': 33, - 'blue': 34, - 'purple': 35, - 'cyan': 36, - 'white': 37, -} + elif is_above_python_2_7(): + super(BaseColorFormatter, self). \ + __init__(fmt, datefmt) # noqa: E122 -COLOR_TAG = "color" -HIGHLIGHT_TAG = "highlight" -ATTRIBUTE_TAG = "attribute" + else: + Formatter. \ + __init__(self, fmt, datefmt) # noqa: E122 + self.style = style + self.colorfmt = colorfmt -RESET = "\033[0m" -FMT_STR = "\033[%dm%s" + def format(self, record): + """Text formatter. + Connects 2 methods. First it extract a level and a colors + assigned to this level in the BaseColorFormatter class. + Second it applied the colors to the text. -def colored(text, color=None, on_color=None, attrs=None): - # It can't support windows system cmd right now! - # TODO: colered output on windows system cmd - if os.name in ('nt', 'ce'): - return text + Args: + record : an instance of a logged event. + Returns: + str : "colored" text (formatted text). + """ - if isinstance(attrs, str): - attrs = [attrs] + if is_above_python_2_7(): + s = super(BaseColorFormatter, self). \ + format(record) - if os.getenv('ANSI_COLORS_DISABLED', None) is None: - if color is not None and isinstance(color, str): - text = FMT_STR % (COLORS.get(color, 0), text) + else: + s = Formatter. \ + format(self, record) - if on_color is not None and isinstance(on_color, str): - text = FMT_STR % (HIGHLIGHTS.get(on_color, 0), text) + color, highlight, attribute = self._parse_color(record) - if attrs is not None: - for attr in attrs: - text = FMT_STR % (ATTRIBUTES.get(attr, 0), text) + return apply_color(s, color, highlight, attrs=attribute) - # keep origin color for tail spaces - text += RESET - return text + def _parse_color(self, record): + # type: (LogRecord) -> (Optional[str], Optional[str], Optional[str]) + """Color formatter based on the logging level. + This method formats the record according to its level + and a color format set for that level. If the level is + not found, then this method will eventually return None. -class BaseColorFormatter(Formatter): + Args: + record : an instance of a logged event. + Returns: + str : Colors. + str : Hightlight tag. + str : Attribute tag. + """ + if self.colorfmt and \ + isinstance(self.colorfmt, dict): - def __init__(self, fmt=None, datefmt=None, colorfmt=None, style="%"): - if is_above_python_3_2(): - super(BaseColorFormatter, self).__init__( - fmt=fmt, datefmt=datefmt, style=style) - elif is_above_python_2_7(): - super(BaseColorFormatter, self).__init__(fmt, datefmt) - else: - Formatter.__init__(self, fmt, datefmt) + level = record.levelname + colors = self.colorfmt.get(level, None) - self.style = style - self.colorfmt = colorfmt + if colors is not None and \ + isinstance(colors, dict): + return (colors.get(COLOR_TAG, None), # noqa: E201 + colors.get(HIGHLIGHT_TAG, None), + colors.get(ATTRIBUTE_TAG, None)) # noqa: E202 + return None, None, None + @deprecated(reason="Will be removed. Use _parse_color(record) instead.") def _parseColor(self, record): """ - color formatter for instance: - { - "logging-levelname": - { - "color":"<COLORS>", - "highlight":"<HIGHLIGHTS>", - "attribute":"<ATTRIBUTES>", - } - } - :param record: - :return: text color, background color, text attribute + Color based on logging level. + See method _parse_color(record). """ - if self.colorfmt and isinstance(self.colorfmt, dict): + return self._parse_color(record) + + +def apply_color(text, # type: str + color=None, # type: Optional[str] + on_color=None, # type: Optional[str] + attrs=None): # type: Optional[Union[str, list]] + # type: (...) -> str + """Applies color codes to the text. + + Args: + text : text to be "colored" (formatted). + color : Color in human-readable format. Defaults to None. + highlight : Hightlight color in human-readable format. + Previously called "on_color". Defaults to None. + attrs : Colors for attribute(s). Defaults to None. + Returns: + str : "colored" text (formatted text). + """ + warn("`on_color` will be replaced with `highlight`.", DeprecationWarning) + highlight = on_color # replace the parameter and remove - level = record.levelname - colors = self.colorfmt.get(level, None) + if os.name in ('nt', 'ce'): + return text - if colors is not None and isinstance(colors, dict): - return colors.get(COLOR_TAG, None), \ - colors.get(HIGHLIGHT_TAG, None), \ - colors.get(ATTRIBUTE_TAG, None) + if isinstance(attrs, str): + attrs = [attrs] - return None, None, None + ansi_disabled = os.getenv('ANSI_COLORS_DISABLED', None) - def format(self, record): + if ansi_disabled is None: - if sys.version_info > (2, 7): - s = super(BaseColorFormatter, self).format(record) - else: - s = Formatter.format(self, record) - color, on_color, attribute = self._parseColor(record) - return colored(s, color, on_color, attrs=attribute) + if color is not None and \ + isinstance(color, str): + text = FMT_STR % (COLORS.get(color, 0), text) + + if highlight is not None and \ + isinstance(highlight, str): + text = FMT_STR % (HIGHLIGHTS.get(highlight, 0), text) + + if attrs is not None: + for attr in attrs: + text = FMT_STR % (ATTRIBUTES.get(attr, 0), text) + + text += RESET # keep origin color for tail spaces + + return text + + +@deprecated(reason="Will be removed. Call apply_color(...) instead.") +def colored(text, color=None, on_color=None, attrs=None): + """ + Format text with color codes. + See method apply_color(text, color, on_color, attrs). + """ + return apply_color(text, color, on_color, attrs) diff --git a/pylog/onaplogging/logWatchDog.py b/pylog/onaplogging/logWatchDog.py index f93dd12..42e8646 100644 --- a/pylog/onaplogging/logWatchDog.py +++ b/pylog/onaplogging/logWatchDog.py @@ -13,69 +13,131 @@ # limitations under the License. import os -import yaml import traceback + from logging import config +from typing import Dict, Optional, Any +from deprecated import deprecated +from warnings import warn + from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler +from watchdog.events import FileSystemEventHandler, FileSystemEvent + +from onaplogging.utils.tools import yaml_to_dict -__all__ = ['patch_loggingYaml'] +__all__ = ['patch_loggingYaml'] # rename after the deprecated name changed -def _yaml2Dict(filename): +class FileEventHandlers(FileSystemEventHandler): + """Handler of the events in the file system. - with open(filename, 'rt') as f: - return yaml.load(f.read()) + Use it to keep and eye on files in the file system. + Extends: + watchdog.events.FileSystemEventHandler + Properties: + filepath : The path to the file to be monitored. + current_config : Defaults to None. + Args: + filepath : The path to the file to be monitored. + """ -class FileEventHandlers(FileSystemEventHandler): + @property + def filepath(self): + # type: () -> str + return self._filepath - def __init__(self, filepath): + @property + def current_config(self): + # type: () -> str + return self.currentConfig # deprecated, replace with _current_config + + @filepath.setter + def filepath(self, value): + # type: (str) -> str + self._filepath = value + + @current_config.setter + def current_config(self, value): + # type: (Dict) -> Dict + self.currentConfig = value + + def __init__(self, filepath): # type: (str) + warn("Attribute currentConfig will be replaced with property" + "current_config. Use current_config instead.") FileSystemEventHandler.__init__(self) + self.filepath = filepath - self.currentConfig = None + self.current_config = None def on_modified(self, event): + # type: (FileSystemEvent) -> None + """Configuration file actualizer. + + When an event occurs in the file system the hadnler's filepath + is taken to update the configuration file. If the actualization + of the config file fails it will keep the old config file. + + Args: + event : Represents an event on the file system. + Raises: + Exception : If the actualization of the config file fails. + """ try: if event.src_path == self.filepath: - newConfig = _yaml2Dict(self.filepath) - print("reload logging configure file %s" % event.src_path) - config.dictConfig(newConfig) - self.currentConfig = newConfig + + new_config = yaml_to_dict(self.filepath) + print("Reloading logging configuration file %s " + % event.src_path) + + config.dictConfig(new_config) + self.current_config = new_config except Exception: traceback.print_exc() print("Reuse the old configuration to avoid this" "exception terminate program") - if self.currentConfig: - config.dictConfig(self.currentConfig) + + if self.current_config: + config.dictConfig(self.current_config) def _yamlConfig(filepath=None, watchDog=None): + # type: (Optional[str], Optional[Any]) -> None + """YAML configuration file loader. - """ - load logging configureation from yaml file and monitor file status + Use it to monitor a file status in a directory. The watchdog can monitor + a YAML file status looking for modifications. If the watchdog is provided + start observing the directory. The new configuration file is saved as + current for the later reuse. + + Args: + filepath : The path to the file to be monitored. Defaults to None. + watchDog : Monitors a YAML file identifier status. Defaults to None. - :param filepath: logging yaml configure file absolute path - :param watchDog: monitor yaml file identifier status - :return: + Raises: + OSError : If the requested file in the filepath is not a file. + Exception : If watchdog observer setup or YAML coversion fails. """ - if os.path.isfile(filepath) is False: - raise OSError("wrong file") + + is_file = os.path.isfile(filepath) + + if is_file is False: + raise OSError("%s is not a file" % (filepath)) dirpath = os.path.dirname(filepath) event_handler = None try: - dictConfig = _yaml2Dict(filepath) - # The watchdog could monitor yaml file status,if be modified - # will send a notify then we could reload logging configuration + dictConfig = yaml_to_dict(filepath) + # Dev note: Will send a notify then we could reload logging config if watchDog: observer = Observer() event_handler = FileEventHandlers(filepath) - observer.schedule(event_handler=event_handler, path=dirpath, + observer.schedule(event_handler=event_handler, + path=dirpath, recursive=False) observer.setDaemon(True) observer.start() @@ -83,14 +145,23 @@ def _yamlConfig(filepath=None, watchDog=None): config.dictConfig(dictConfig) if event_handler: - # here we keep the correct configuration for reusing event_handler.currentConfig = dictConfig except Exception: traceback.print_exc() -def patch_loggingYaml(): - # The patch to add yam config forlogginf and runtime - # reload logging when modify yaml file +def patch_logging_yaml(): + # type: () -> None + """YAML configuration patch. + + Adds the YAML configuration file loader + to logging.config module during runtime. + """ config.yamlConfig = _yamlConfig + + +@deprecated(reason="Will be removed. Call patch_logging_yaml() instead.") +def patch_loggingYaml(): + """See patch_logging_yaml()""" + patch_logging_yaml() diff --git a/pylog/onaplogging/marker/marker.py b/pylog/onaplogging/marker/marker.py index 5414e21..17a3328 100644 --- a/pylog/onaplogging/marker/marker.py +++ b/pylog/onaplogging/marker/marker.py @@ -14,10 +14,30 @@ import abc +from typing import Iterable, List, Optional, Union, Iterator +from deprecated import deprecated +from warnings import warn +from logging import LogRecord + MARKER_TAG = "marker" class Marker(object): + """Abstract class for defining the marker structure. + + TODO: + after deprecated child methods are removed, rename them here. + Extends: + object + Method list: + getName + addChild + addChilds + removeChild + contains + Raises: + NotImplementedError + """ __metaclass__ = abc.ABCMeta @@ -51,92 +71,235 @@ class Marker(object): class BaseMarker(Marker): + """Basic marker class. + + It is a marker with base functionalities that add sub-level markers and + check if another marker exists as the parent itself or as its child. + + Extends: + Marker + Properties: + name : The name of the marker. + children (list) : The list of all children (sub-level) markers. + Arguments: + name (str) : The name of the marker. + Methods: + getName : Returns the name of the marker. + addChild : Adds a sub-level marker. + addChilds : Adds a list of sub-level markers. + removeChild : Removes a specified sub-level marker. + contains : Checks if a sub-level marker exists. + """ + + @property + def name(self): + # type: () -> str + """Name of the parent marker.""" + return self.__name + + @property + def children(self): + # type: () -> List[Marker] + """Child markers of one parent marker.""" + return self.__childs + + @name.setter + def name(self, value): + # type: (str) -> None + self.__name = value + + @children.setter + def children(self, value): + # type: (List[Marker]) -> None + self.__childs = value + + def __init__(self, name): # type: (str) + """ + Raises: + TypeError : If the `name` parameter is not a string. + ValueError : If the `name` parameter is an empty string. + """ - def __init__(self, name): super(BaseMarker, self).__init__() + if not isinstance(name, str): raise TypeError("not str type") + if name == "": raise ValueError("empty value") + + warn("Attribute `__childs` is replaced by the property `children`." + "Use children instead.", DeprecationWarning) + self.__name = name self.__childs = [] - def getName(self): - return self.__name + def add_child(self, marker): + # type: (Marker) -> None + """Append a marker to child markers. - def __iter__(self): - return iter(self.__childs) + Use this method to describe a different level of logs. For example, + error log would use the ERROR marker. However it's possible to + create a, for instance, TYPE_ERROR to mark type related events. + In this case TYPE_ERROR will be a child of parent ERROR. - def __eq__(self, other): + Args: + marker : marker describing a different log level. + Raises: + TypeError : if the marker object has different type. + """ - if not isinstance(other, Marker): - return False - if id(self) == id(other): - return True + if not isinstance(marker, Marker): + raise TypeError("Bad marker type. \ + Can only add markers of type Marker. \ + Type %s was passed." % type(marker)) - return self.__name == other.getName() + if self == marker: + return - def __hash__(self): - return hash(self.__name) + if marker not in self.children: + self.children.append(marker) - def contains(self, item=None): + def add_children(self, markers): + # type: (Iterable[List]) -> None + """ Append a list of markers to child markers. + + Args: + markers : An iterable object, containing markers. + Raises: + Exception : If `marker` parameter is not iterable. + """ + + try: + iter(markers) + except Exception as e: + raise e + + for marker in markers: + self.children.append(marker) + + def remove_child(self, marker): + # type: (Marker) -> None + """Use this method to remove a marker from the children list. - if isinstance(item, Marker): - if item == self: + Args: + marker : A child marker object. + Raises: + TypeError: if the marker object has different type. + """ + + if not isinstance(marker, Marker): + raise TypeError("Bad marker type. \ + Can only add markers of type Marker. \ + Type %s was passed." % type(marker)) + + if marker in self.children: + self.children.remove(marker) + + def contains(self, item=None): + # type: (Optional[Union[Marker, str]]) -> bool + """ + Use it to check if a marker exists as a parent itself or its chidren. + + Args: + item : A child marker object. Defaults to None. + Returns: + bool : True if the marker exists. + """ + + warn("`item` argument will be replaced with `marker`. " + "Default value None will be removed.", + DeprecationWarning) + marker = item + + if isinstance(marker, Marker): + if marker == self: return True return len(list(filter( - lambda x: x == item, self.__childs))) > 0 + lambda x: x == marker, self.children))) > 0 - elif isinstance(item, str): - if item == self.__name: + elif isinstance(marker, str): + if marker == self.name: return True return len(list(filter( - lambda x: x.__name == item, self.__childs))) > 0 + lambda x: x.name == marker, self.children))) > 0 return False - def addChild(self, item): - if not isinstance(item, Marker): - raise TypeError("can only add (not %s) marker type" - % type(item)) - if self == item: - return - if item not in self.__childs: - self.__childs.append(item) + def __iter__(self): + # type: () -> Iterator[List[Marker]] + return iter(self.__childs) + + def __hash__(self): + # type (): -> int + return hash(self.__name) + def __eq__(self, other): + # type: (Marker) -> bool + if not isinstance(other, Marker): + return False + if id(self) == id(other): + return True + + return self.__name == other.getName() + + @deprecated(reason="Will be removed. Call the `name` property instead.") + def getName(self): + """Class attribute getter.""" + return self.name + + @deprecated(reason="Will be removed. Call add_children(markers) instead.") def addChilds(self, childs): - try: - iter(childs) - except Exception as e: - raise e + """Add a list of sub-level markers. See add_children(markers)""" + self.add_children(childs) - for item in childs: - self.addChild(item) + @deprecated(reason="Will be removed. Call add_child(marker) instead.") + def addChild(self, item): + """Add a sub-level marker. See add_child(marker)""" + self.add_child(item) + @deprecated(reason="Will be removed. Call remove_child(marker) instead.") def removeChild(self, item): - if not isinstance(item, Marker): - raise TypeError("can only add (not %s) marker type" - % type(item)) - if item in self.__childs: - self.__childs.remove(item) + """Remove a sub-level marker. See remove_child(marker)""" + self.remove_child(item) +@deprecated(reason="Will be removed. " + "Call match_marker(record, marker_to_match) instead.") def matchMarkerHelp(record, markerToMatch): - - marker = getattr(record, MARKER_TAG, None) - - if marker is None or markerToMatch is None: + """See match_marker(record, marker_to_match).""" + return match_markers(record, markerToMatch) + + +def match_markers(record, marker_to_match): + # type: (LogRecord, Union[Marker, List]) -> bool + """ + Use this method to match a marker (or a list of markers) with a LogRecord + record. + + Args: + record : a record that may contain a marker. + markerToMatch : a marker or a list of markers. + Raises: + Exception : if match check went wrong. + Returns: + bool : whether the check can be done or the marker is found. + """ + record_marker = getattr(record, MARKER_TAG, None) + + if record_marker is None or \ + marker_to_match is None: return False - if not isinstance(marker, Marker): + if not isinstance(record_marker, Marker): return False try: - if isinstance(markerToMatch, list): + if isinstance(marker_to_match, list): return len(list(filter( - lambda x: marker.contains(x), markerToMatch))) > 0 + lambda x: record_marker.contains(x), marker_to_match))) > 0 - return marker.contains(markerToMatch) + return record_marker.contains(marker_to_match) except Exception as e: raise e diff --git a/pylog/onaplogging/marker/markerFactory.py b/pylog/onaplogging/marker/markerFactory.py index 0705235..a0e9887 100644 --- a/pylog/onaplogging/marker/markerFactory.py +++ b/pylog/onaplogging/marker/markerFactory.py @@ -14,12 +14,32 @@ import abc import threading + +from deprecated import deprecated +from warnings import warn +from typing import Dict, Optional + +from .marker import Marker from .marker import BaseMarker lock = threading.RLock() class IMarkerFactory(object): + """Abstract marker factory for defining structure. + + TODO: + after deprecated child methods are removed, rename them here. + Extends: + object + Method list: + getMarker + deleteMarker + exist + Raises: + NotImplementedError + """ + __metaclass__ = abc.ABCMeta @abc.abstractmethod @@ -36,39 +56,111 @@ class IMarkerFactory(object): class MarkerFactory(IMarkerFactory): + """A factory class maganing every marker. + + It is designed to check the existance, create and remove single markers. + This class follows a singleton pattern - only one instance can be created. + + Extends: + IMarkerFactory + Properties: + marker_map : a map of existing markers. + Attributes: + _instance : a marker factory instance. + Methods: + getMarker : creates a new marker or returns an available one. + deleteMarker : removes a specific marker. + exist : checks if a specific marker exists. + """ _instance = None _marker_map = {} - def __new__(cls, *args, **kwargs): - - if cls._instance is None: - cls._instance = super(MarkerFactory, cls).__new__(cls) + @property + def marker_map(self): + # type: () -> Dict + if not hasattr(self, '_marker_map'): + self._marker_map = {} + return self._marker_map + + def get_marker(self, name=None): + # type: (Optional[str]) -> Marker + """ + Use it to get any marker by its name. If it doesn't exist - it + will create a new marker that will be added to the factory. + Blocks the thread while executing. + + Args: + name : A marker name. Defaults to None. + Raises: + ValueError : If `name` is None. + Returns: + Marker : A found or just newly created marker. + """ + + if name is None: + raise ValueError("Marker name is None. Must have a str value.") - return cls._instance + lock.acquire() - def getMarker(self, marker_name=None): - if marker_name is None: - raise ValueError("not empty") + marker = self.marker_map.get(name, None) - lock.acquire() - marker = self._marker_map.get(marker_name, None) if marker is None: - marker = BaseMarker(name=marker_name) - self._marker_map[marker_name] = marker + marker = BaseMarker(name) + self.marker_map[name] = marker + lock.release() return marker - def deleteMarker(self, marker_name=None): + def delete_marker(self, name=None): + # type: (Optional[str]) -> bool + """ + Args: + name: A marker name. Defaults to None. + Returns: + bool: The status of deletion. + """ + lock.acquire() - if self.exist(marker_name): - del self._marker_map[marker_name] + exists = self.exists(name) + if exists: + del self.marker_map[name] return True lock.release() + return False + def exists(self, name=None): + # type: (Optional[str]) -> bool + """ + Checks whether the search for a marker returns None and returns the + status of the operation. + + Args: + name: marker name. Defaults to None. + Returns: + bool: status of whether the marker was found. + """ + marker = self.marker_map.get(name, None) + return marker is not None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(MarkerFactory, cls).__new__(cls) + + warn("_marker_map attribute will be replaced by marker_map property.", + DeprecationWarning) + return cls._instance + + @deprecated(reason="Will be removed. Call exists(name) instead.") def exist(self, marker_name=None): + return self.exists(marker_name) - return self._marker_map.get( - marker_name, None) is not None + @deprecated(reason="Will be removed. Call get_marker(name) instead.") + def getMarker(self, marker_name=None): + return self.get_marker(marker_name) + + @deprecated(reason="Will be removed. Call delete_marker(name) instead.") + def deleteMarker(self, marker_name=None): + return self.delete_marker(marker_name) diff --git a/pylog/onaplogging/marker/markerFilter.py b/pylog/onaplogging/marker/markerFilter.py index 4f49884..a381d8e 100644 --- a/pylog/onaplogging/marker/markerFilter.py +++ b/pylog/onaplogging/marker/markerFilter.py @@ -12,21 +12,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys -from logging import Filter -from .marker import matchMarkerHelp +from logging import Filter, LogRecord +from warnings import warn +from typing import List, Optional, Union + +from onaplogging.utils.system import is_above_python_2_7 + +from .marker import match_markers, Marker class MarkerFilter(Filter): + """Marker filtering. + + Extends: + logging.Filter + Properties: + marker_to_match (Marker/list): a marker of list of markers. + Methods + filter: Filter records by the current filter marker(s). + """ + + @property + def markers_to_match(self): + # type: () -> Union[Marker, List[Marker]] + return self.markersToMatch # TODO renamed - deprecated - def __init__(self, name="", markers=None): - if sys.version_info > (2, 7): + @markers_to_match.setter + def markers_to_match(self, value): + # type: ( Union[Marker, List[Marker]] ) -> None + self.markersToMatch = value + + def __init__(self, + name="", # type: str + markers=None): # type: Optional[Union[Marker, List[Marker]]] + + if is_above_python_2_7(): super(MarkerFilter, self).__init__(name) + else: Filter.__init__(self, name) - self.markerToMatch = markers + warn("markersToMatch attribute will be replaced by a property. " + "Use markers_to_match property instead.", DeprecationWarning) + self.markers_to_match = markers def filter(self, record): - # compare filter's markers with record's marker - return matchMarkerHelp(record, self.markerToMatch) + # type: (LogRecord) -> bool + """Filter by looking for a marker match. + + Args: + record: A record to match with the filter(s). + Returns: + bool: Whether the record matched with the filter(s) + """ + return match_markers(record, self.markers_to_match) diff --git a/pylog/onaplogging/marker/markerHandler.py b/pylog/onaplogging/marker/markerHandler.py index e9ce810..36934a8 100644 --- a/pylog/onaplogging/marker/markerHandler.py +++ b/pylog/onaplogging/marker/markerHandler.py @@ -12,40 +12,118 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys +from logging import LogRecord from logging.handlers import SMTPHandler -from .marker import matchMarkerHelp +from typing import Tuple, List, Optional, Union + +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 + +from .marker import match_markers, Marker class MarkerNotifyHandler(SMTPHandler): + """Handler for email notification. + + Wraps logging.handler.SMTPHandler and extends it by sending only such + notifications which contain certain markers. + + Extends: + SMTPHandler + Property: + markers: A marker or a list of markers. + Args: + mailhost: A (host, port) tuple. + fromaddr: The sender of the email notification. + toaddrs: Email notification recepient(s). + subject: Email subject. + credentials: A (username, password) tuple. + secure: For example (TLS). It is used when the + credentials are supplied. + timout: Default is 5.0 seconds. Python version 3.2+ + markers: A marker or a list of markers. + """ + + @property + def markers(self): + # type: () -> Union[Marker, List[Marker]] + return self._markers + + @markers.setter + def markers(self, value): + # type: ( Union[Marker, List[Marker]] ) - None + self._markers = value + + def __init__(self, + mailhost, # type: Tuple + fromaddr, # type: str + toaddrs, # type: Union[List[str], str] + subject, # type: Tuple + credentials=None, # type: Tuple + secure=None, # type: Optional[Tuple] + timeout=5.0, # type: Optional[float] + markers=None # type: Optional[Union[Marker, List[Marker]]] + ): + + if is_above_python_3_2(): + super(MarkerNotifyHandler, self). \ + __init__( # noqa: E122 + mailhost, + fromaddr, + toaddrs, + subject, + credentials, + secure, + timeout) + + elif is_above_python_2_7(): + super(MarkerNotifyHandler, self). \ + __init__( # noqa: E122 + mailhost, + fromaddr, + toaddrs, + subject, + credentials, + secure) - def __init__(self, mailhost, fromaddr, toaddrs, subject, - credentials=None, secure=None, timeout=5.0, markers=None): - - if sys.version_info > (3, 2): - super(MarkerNotifyHandler, self).__init__( - mailhost, fromaddr, toaddrs, subject, - credentials, secure, timeout) - elif sys.version_info > (2, 7): - super(MarkerNotifyHandler, self).__init__( - mailhost, fromaddr, toaddrs, subject, - credentials, secure) else: SMTPHandler.__init__(self, - mailhost, fromaddr, toaddrs, subject, - credentials, secure) + mailhost, + fromaddr, + toaddrs, + subject, + credentials, + secure) self.markers = markers def handle(self, record): + # type: (LogRecord) -> bool + """ + Handle a LogRecord record. Send an email notification. + """ + return self.send_notification(record) + + def send_notification(self, record): + # type: (LogRecord) -> bool + """Email notification handler. - if self.markers is None: + Matches the record with the specific markers set for email + notifications. Sends an email notification if that marker(s) matched. + + Args: + record (LogRecord): A record that might contain a marker. + Returns: + bool: Whether a record was passed for emission (to be sent). + """ + + if hasattr(self, "markers") and \ + self.markers is None: return False - if matchMarkerHelp(record, self.markers): - if sys.version_info > (2, 7): + if match_markers(record, self.markers): + + if is_above_python_2_7(): return super(MarkerNotifyHandler, self).handle(record) - else: - return SMTPHandler.handle(self, record) + return SMTPHandler.handle(self, record) return False diff --git a/pylog/onaplogging/markerFormatter.py b/pylog/onaplogging/markerFormatter.py index a322e29..d0da695 100644 --- a/pylog/onaplogging/markerFormatter.py +++ b/pylog/onaplogging/markerFormatter.py @@ -12,57 +12,116 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import logging -from .marker import MARKER_TAG -from .marker import Marker +from logging import LogRecord +from typing import Optional + +from onaplogging.utils.styles import MARKER_OPTIONS +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 + +from .marker import Marker, MARKER_TAG from .colorFormatter import BaseColorFormatter class MarkerFormatter(BaseColorFormatter): + """Formats coloring styles based on a marker. - def __init__(self, fmt=None, datefmt=None, colorfmt=None, style='%'): + If `fmt` is not supplied, the `style` is used. - if sys.version_info > (3, 2): - super(MarkerFormatter, self).__init__( - fmt=fmt, datefmt=datefmt, colorfmt=colorfmt, style=style) - elif sys.version_info > (2, 7): - super(MarkerFormatter, self).__init__( - fmt=fmt, datefmt=datefmt, colorfmt=colorfmt) - else: - BaseColorFormatter.__init__(self, fmt, datefmt, colorfmt) + Extends: + BaseColorFormatter + Properties: + marker_tag: a marker to be applied. + temp_fmt : keeps initial format to be reset to after formatting. + Args: + fmt : human-readable format. Defaults to None. + datefmt : ISO8601-like (or RFC 3339-like) format. Defaults to None. + colorfmt : color schemas for logging levels. Defaults to None. + style : '%', '{' or '$' formatting. Defaults to '%'. + Added in Python 3.2. + """ + + @property + def marker_tag(self): + # type: () -> str + return self._marker_tag + + @property + def temp_fmt(self): + # type: () -> str + return self._temp_fmt + + @marker_tag.setter + def marker_tag(self, value): + # type: (str) -> None + self._marker_tag = value - self._marker_tag = "%(marker)s" + @temp_fmt.setter + def temp_fmt(self, value): + # type: (str) -> None + self._temp_fmt = value - if self.style == "{": - self._marker_tag = "{marker}" - elif self.style == "$": - self._marker_tag = "${marker}" + def __init__(self, + fmt=None, # type: Optional[str] + datefmt=None, # type: Optional[str] + colorfmt=None, # type: Optional[dict] + style='%'): # type: Optional[str] - self._tmpFmt = self._fmt + if is_above_python_3_2(): + super(MarkerFormatter, self).\ + __init__(fmt=fmt, # noqa: E122 + datefmt=datefmt, + colorfmt=colorfmt, + style=style) # added in Python 3.2+ + + elif is_above_python_2_7(): + super(MarkerFormatter, self).\ + __init__(fmt=fmt, # noqa: E122 + datefmt=datefmt, + colorfmt=colorfmt) + + else: + BaseColorFormatter.\ + __init__(self, fmt, datefmt, colorfmt) # noqa: E122 + + self.marker_tag = MARKER_OPTIONS[self.style] + self.temp_fmt = self._fmt def format(self, record): + # type: (LogRecord) -> str + """Marker formatter. + + Use it to apply the marker from the LogRecord record to the formatter + string `fmt`. + Args: + record : an instance of a logged event. + Returns: + str : "colored" text (formatted text). + """ try: - if self._fmt.find(self._marker_tag) != -1 \ - and hasattr(record, MARKER_TAG): + + if self._fmt.find(self.marker_tag) != -1 and \ + hasattr(record, MARKER_TAG): marker = getattr(record, MARKER_TAG) if isinstance(marker, Marker): - self._fmt = self._fmt.replace( - self._marker_tag, marker.getName()) - elif self._fmt.find(self._marker_tag) != -1 \ - and not hasattr(record, MARKER_TAG): + self._fmt = self._fmt.replace(self.marker_tag, + marker.name) - self._fmt = self._fmt.replace(self._marker_tag, "") + elif self._fmt.find(self.marker_tag) != -1 and \ + not hasattr(record, MARKER_TAG): + self._fmt = self._fmt.replace(self.marker_tag, "") - if sys.version_info > (3, 2): - self._style = logging._STYLES[self.style][0](self._fmt) + if is_above_python_3_2(): + StylingClass = logging._STYLES[self.style][0] + self.style = StylingClass(self._fmt) - if sys.version_info > (2, 7): + if is_above_python_2_7(): + # includes Python 3.2+ style attribute return super(MarkerFormatter, self).format(record) else: return BaseColorFormatter.format(self, record) finally: - self._fmt = self._tmpFmt + self._fmt = self.temp_fmt diff --git a/pylog/onaplogging/markerLogAdaptor.py b/pylog/onaplogging/markerLogAdaptor.py index e901758..2c1e5df 100644 --- a/pylog/onaplogging/markerLogAdaptor.py +++ b/pylog/onaplogging/markerLogAdaptor.py @@ -12,74 +12,173 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys from logging import LoggerAdapter from threading import RLock from functools import wraps -from .marker import MARKER_TAG -from .marker import Marker +from deprecated import deprecated +from typing import Dict, Callable + +from onaplogging.utils.system import is_above_python_3_2 + +from .marker import Marker, MARKER_TAG from .mdcContext import _getmdcs lock = RLock() -def addMarker(func): +def add_marker(func): + # type: ( Callable[[Marker, str], None] ) -> Callable[[Marker, str], None] + """Marker decorator. + + Requests a blocking acquisition of the thread. Sets the marker + as the logger's marker and delegates a call to the underlying + logger with contextual information. Next it removes the marker + and releases the thread. + + Args: + func : a method supplied with a logging marker. + Raises: + TypeError : the marker type is not `Marker`. + Exception : `extra` doesn't exist or MARKER_TAG is in `extra`. + Returns: + method: decorated method. + """ @wraps(func) def wrapper(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> Callable[[Marker, str], None] + lock.acquire() + if not isinstance(marker, Marker): - raise TypeError("not marker type %s" - % type(marker)) + raise TypeError("Passed a marker of type %s. \ + Should have the type %s." + % type(marker), "Marker") + + if self.extra and \ + MARKER_TAG in self.extra: + raise Exception("Can't add 'marker' in extra - either extra \ + exists or MARKER_TAG is alredy in extra") - if self.extra and MARKER_TAG in self.extra: - raise Exception("cann't add 'marker' in extra") setattr(self.logger, MARKER_TAG, marker) + func(self, marker, msg, *args, **kwargs) + if hasattr(self.logger, MARKER_TAG): delattr(self.logger, MARKER_TAG) + lock.release() + return wrapper +@deprecated(reason="@addMarker is deprecated. Use @add_marker instead.") +def addMarker(func): + """Decorator. See new decorator add_marker(func).""" + add_marker(func) + + class MarkerLogAdaptor(LoggerAdapter): + """Contextual loggin adapter. - def process(self, msg, kwargs): + Specifies contextual information in logging output. Takes a logger and a + dictionary-like object `extra` for providing contextual information. + + An example of the extra contextual information: + extra = {'app_name':'Marker Logging'} + + Extends: + logging.LoggerAdapter + """ - if sys.version_info > (3, 2): + def process(self, msg, kwargs): + # type: (str, Dict) + """Logging call processor. + + Takes a logging message and keyword arguments to provide cotextual + information. + + Args: + msg : Logging information. + kwargs : Contextual information. + Returns: + str : Logging message. + dict : modified (or not) contextual information. + """ + if is_above_python_3_2(): kwargs['extra'] = _getmdcs(self.extra) else: kwargs['extra'] = self.extra return msg, kwargs - @addMarker - def infoMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def info_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with an informational call.""" self.info(msg, *args, **kwargs) - @addMarker - def debugMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def debug_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a debug call.""" self.debug(msg, *args, **kwargs) - @addMarker - def warningMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def warning_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a warning call.""" self.warning(msg, *args, **kwargs) - @addMarker - def errorMarker(self, marker, msg, *args, **kwargs): - + @add_marker + def error_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with an error call.""" self.error(msg, *args, **kwargs) - @addMarker - def exceptionMarker(self, marker, msg, *arg, **kwargs): - self.exception(msg, *arg, **kwargs) + @add_marker + def exception_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with an exceptional call.""" + self.exception(msg, *args, **kwargs) + + @add_marker + def critical_marker(self, marker, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a critical call.""" + self.critical(msg, *args, **kwargs) + + @add_marker + def log_marker(self, marker, level, msg, *args, **kwargs): + # type: (Marker, str) -> None + """Provide the logger with a log call.""" + self.log(marker, level, msg, *args, **kwargs) + + @deprecated(reason="infoMarker(...) is replaced with info_marker(...).") + def infoMarker(self, marker, msg, *args, **kwargs): + self.info_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="debugMarker(...) is replaced with debug_marker(...).") + def debugMarker(self, marker, msg, *args, **kwargs): + self.debug_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="warningMarker(...) replaced, use warning_marker(...).") + def warningMarker(self, marker, msg, *args, **kwargs): + self.warning_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="errorMarker(...) is replaced with error_marker(...).") + def errorMarker(self, marker, msg, *args, **kwargs): + self.error_marker(marker, msg, *args, **kwargs) + + @deprecated(reason="exceptionMarker(...) replaced," + " use exception_marker(...).") + def exceptionMarker(self, marker, msg, *args, **kwargs): + self.exception_marker(marker, msg, *args, **kwargs) - @addMarker - def criticalMarker(self, marker, msg, *arg, **kwargs): - self.critical(msg, *arg, **kwargs) + @deprecated(reason="criticalMarker(...) is replaced, " + "use critical_marker(...).") + def criticalMarker(self, marker, msg, *args, **kwargs): + self.critical_marker(marker, msg, *args, **kwargs) - @addMarker - def logMarker(self, marker, level, msg, *arg, **kwargs): - self.log(level, msg, *arg, **kwargs) + @deprecated(reason="logMarker(...) is replaced with info_marker(...).") + def logMarker(self, marker, level, msg, *args, **kwargs): + self.log_marker(marker, level, msg, *args, **kwargs) diff --git a/pylog/onaplogging/mdcContext.py b/pylog/onaplogging/mdcContext.py index ecdc2d9..c1852b3 100644 --- a/pylog/onaplogging/mdcContext.py +++ b/pylog/onaplogging/mdcContext.py @@ -19,9 +19,16 @@ import os import traceback import sys import functools -from .marker import Marker -from .marker import MARKER_TAG +from deprecated import deprecated +from typing import Dict, Optional, Any, Callable, List, Tuple +from logging import LogRecord + +from onaplogging.utils.system import is_above_python_3_2 + +from .marker import Marker, MARKER_TAG + +# TODO change to patch_logging_mdc after deprecated method is removed __all__ = ['patch_loggingMDC', 'MDC'] _replace_func_name = ['info', 'critical', 'fatal', 'debug', @@ -29,123 +36,186 @@ _replace_func_name = ['info', 'critical', 'fatal', 'debug', 'handle', 'findCaller'] -class MDCContext(threading.local): +def fetchkeys(func): # type: Callable[[str, List, Dict], None] + # type: (...) -> Callable[[str, List, Dict], None] + """MDC decorator. + + Fetchs contextual information from a logging call. + Wraps by adding MDC to the `extra` field. Executes + the call with the updated contextual information. """ - A Thread local instance to storage mdc values + + @functools.wraps(func) + def replace(*args, **kwargs): + # type: () -> None + kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None)) + func(*args, **kwargs) + + return replace + + +class MDCContext(threading.local): + """A Thread local instance that stores MDC values. + + Is initializ with an empty dictionary. Manages that + dictionary to created Mapped Diagnostic Context. + + Extends: + threading.local + Property: + local_dict : a placeholder for MDC keys and values. """ - def __init__(self): + @property + def local_dict(self): + # type: () -> Dict + return self._local_dict + + @local_dict.setter + def local_dict(self, value): + # type: (Dict) -> None + self._local_dict = value + + def __init__(self): super(MDCContext, self).__init__() - self._localDict = {} + self.local_dict = {} def get(self, key): - - return self._localDict.get(key, None) + # type: (str) -> Any + """Retrieve a value by key.""" + return self.local_dict.get(key, None) def put(self, key, value): - - self._localDict[key] = value + # type: (str, Any) -> None + """Insert or update a value by key.""" + self.local_dict[key] = value def remove(self, key): - - if key in self._localDict: - del self._localDict[key] + # type: (str) -> None + """Remove a value by key, if exists.""" + if key in self.local_dict: + del self.local_dict[key] def clear(self): + # type: () -> None + """Empty the MDC dictionary.""" + self.local_dict.clear() - self._localDict.clear() - + @deprecated(reason="Use local_mdc property instead.") def result(self): + """Getter for the MDC dictionary.""" + return self.local_dict - return self._localDict + def empty(self): + # type: () -> bool + """Checks whether the local dictionary is empty.""" + return self.local_dict == {} or \ + self.local_dict is None + @deprecated(reason="Will be replaced. Use empty() instead.") def isEmpty(self): - - return self._localDict == {} or self._localDict is None + """See empty().""" + return self.empty() MDC = MDCContext() -def fetchkeys(func): - - @functools.wraps(func) - def replace(*args, **kwargs): - kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None)) - func(*args, **kwargs) - return replace - - def _getmdcs(extra=None): + # type: (Optional[Dict]) -> Dict """ - Put mdc dict in logging record extra filed with key 'mdc' - :param extra: dict - :return: mdc dict + Puts an MDC dict in the `extra` field with key 'mdc'. This provides + the contextual information with MDC. + + Args: + extra : Contextual information. Defaults to None. + Raises: + KeyError : a key from extra is attempted to be overwritten. + Returns: + dict : contextual information named `extra` with MDC. """ - if MDC.isEmpty(): + if MDC.empty(): return extra - mdc = MDC.result() + mdc = MDC.local_dict if extra is not None: for key in extra: - # make sure extra key dosen't override mdckey - if key in mdc or key == 'mdc': + if key in mdc or \ + key == 'mdc': raise KeyError("Attempt to overwrite %r in MDC" % key) else: extra = {} extra['mdc'] = mdc - del mdc + return extra @fetchkeys def info(self, msg, *args, **kwargs): - + # type: (str) -> None + """If INFO enabled, deletage an info call with MDC.""" if self.isEnabledFor(logging.INFO): self._log(logging.INFO, msg, args, **kwargs) @fetchkeys def debug(self, msg, *args, **kwargs): + # type: (str) -> None + """If DEBUG enabled, deletage a debug call with MDC.""" if self.isEnabledFor(logging.DEBUG): self._log(logging.DEBUG, msg, args, **kwargs) @fetchkeys def warning(self, msg, *args, **kwargs): + # type: (str) -> None + """If WARNING enabled, deletage a warning call with MDC.""" if self.isEnabledFor(logging.WARNING): self._log(logging.WARNING, msg, args, **kwargs) @fetchkeys def exception(self, msg, *args, **kwargs): - + # type: (str) -> None + """Deletage an exception call and set exc_info code to 1.""" kwargs['exc_info'] = 1 self.error(msg, *args, **kwargs) @fetchkeys def critical(self, msg, *args, **kwargs): - + # type: (str) -> None + """If CRITICAL enabled, deletage a critical call with MDC.""" if self.isEnabledFor(logging.CRITICAL): self._log(logging.CRITICAL, msg, args, **kwargs) @fetchkeys def error(self, msg, *args, **kwargs): + # type: (str) -> None + """If ERROR enabled, deletage an error call with MDC.""" if self.isEnabledFor(logging.ERROR): self._log(logging.ERROR, msg, args, **kwargs) @fetchkeys def log(self, level, msg, *args, **kwargs): + # type: (int, str) -> None + """ + If a specific logging level enabled and the code is represented + as an integer value, delegate the call to the underlying logger. + + Raises: + TypeError: if the logging level code is not an integer. + """ if not isinstance(level, int): if logging.raiseExceptions: - raise TypeError("level must be an integer") + raise TypeError("Logging level code must be an integer." + "Got %s instead." % type(level)) else: return @@ -154,55 +224,80 @@ def log(self, level, msg, *args, **kwargs): def handle(self, record): - + # type: (LogRecord) -> None cmarker = getattr(self, MARKER_TAG, None) if isinstance(cmarker, Marker): setattr(record, MARKER_TAG, cmarker) - if (not self.disabled) and self.filter(record): + if not self.disabled and \ + self.filter(record): self.callHandlers(record) def findCaller(self, stack_info=False): + # type: (bool) -> Tuple + """ + Find the stack frame of the caller so that we can note the source file + name, line number and function name. Enhances the logging.findCaller(). + """ + + frame = logging.currentframe() - f = logging.currentframe() - if f is not None: - f = f.f_back + if frame is not None: + frame = frame.f_back rv = "(unkown file)", 0, "(unknow function)" - while hasattr(f, "f_code"): - co = f.f_code + + while hasattr(frame, "f_code"): + co = frame.f_code filename = os.path.normcase(co.co_filename) # jump through local 'replace' func frame - if filename == logging._srcfile or co.co_name == "replace": - f = f.f_back + if filename == logging._srcfile or \ + co.co_name == "replace": + + frame = frame.f_back continue - if sys.version_info > (3, 2): + + if is_above_python_3_2(): + sinfo = None if stack_info: + sio = io.StringIO() sio.write("Stack (most recent call last):\n") - traceback.print_stack(f, file=sio) + traceback.print_stack(frame, file=sio) sinfo = sio.getvalue() + if sinfo[-1] == '\n': sinfo = sinfo[:-1] + sio.close() - rv = (co.co_filename, f.f_lineno, co.co_name, sinfo) + rv = (co.co_filename, frame.f_lineno, co.co_name, sinfo) + else: - rv = (co.co_filename, f.f_lineno, co.co_name) + rv = (co.co_filename, frame.f_lineno, co.co_name) break return rv -def patch_loggingMDC(): - """ - The patch to add MDC ability in logging Record instance at runtime +def patch_logging_mdc(): + # type: () -> None + """MDC patch. + + Sets MDC in a logging record instance at runtime. """ localModule = sys.modules[__name__] + for attr in dir(logging.Logger): if attr in _replace_func_name: newfunc = getattr(localModule, attr, None) if newfunc: setattr(logging.Logger, attr, newfunc) + + +@deprecated(reason="Will be removed. Call patch_logging_mdc() instead.") +def patch_loggingMDC(): + """See patch_logging_ymdc().""" + patch_logging_mdc() diff --git a/pylog/onaplogging/mdcformatter.py b/pylog/onaplogging/mdcformatter.py index 4cacbe8..442f0ad 100644 --- a/pylog/onaplogging/mdcformatter.py +++ b/pylog/onaplogging/mdcformatter.py @@ -13,26 +13,60 @@ # limitations under the License. import logging +from logging import LogRecord +from typing import Mapping, List, Dict, Callable +from deprecated import deprecated + +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 +from onaplogging.utils.styles import MDC_OPTIONS + from .markerFormatter import MarkerFormatter -from .utils import is_above_python_2_7, is_above_python_3_2 class MDCFormatter(MarkerFormatter): - """ - A custom MDC formatter to prepare Mapped Diagnostic Context - to enrich log message. + """A custom MDC formatter. + + Prepares Mapped Diagnostic Context to enrich log message. If `fmt` is not + supplied, the `style` is used. + + Extends: + MarkerFormatter + Args: + fmt : Built-in format string containing standard Python + %-style mapping keys in human-readable format. + mdcFmt : MDC format with '{}'-style mapping keys. + datefmt : Date format. + colorfmt : colored output with an ANSI terminal escape code. + style : style mapping keys in Python 3.x. """ - def __init__(self, fmt=None, mdcfmt=None, - datefmt=None, colorfmt=None, style="%"): - """ - :param fmt: build-in format string contains standard - Python %-style mapping keys - :param mdcFmt: mdc format with '{}'-style mapping keys - :param datefmt: Date format to use - :param colorfmt: colored output with ANSI escape code on terminal - :param style: style mapping keys in python3 - """ + @property + def mdc_tag(self): + # type: () -> str + return self._mdc_tag + + @property + def mdcfmt(self): + # type: () -> str + return self._mdcFmt + + @mdc_tag.setter + def mdc_tag(self, value): + # type: (str) -> str + self._mdc_tag = value + + @mdcfmt.setter + def mdcfmt(self, value): + # type: (str) -> str + self._mdc_tag = value + + def __init__(self, + fmt=None, # type: str + mdcfmt=None, # type: str + datefmt=None, # type: str + colorfmt=None, # type: str + style="%"): # type: str + if is_above_python_3_2(): super(MDCFormatter, self).__init__(fmt=fmt, datefmt=datefmt, @@ -43,114 +77,149 @@ class MDCFormatter(MarkerFormatter): datefmt=datefmt, colorfmt=colorfmt) else: - MarkerFormatter.__init__(self, fmt, datefmt, colorfmt) + MarkerFormatter.\ + __init__(self, fmt, datefmt, colorfmt) # noqa: E122 - self._mdc_tag = "%(mdc)s" - if self.style == "{": - self._mdc_tag = "{mdc}" - elif self.style == "$": - self._mdc_tag = "${mdc}" - - if mdcfmt: - self._mdcFmt = mdcfmt - else: - self._mdcFmt = '{reqeustID}' + self._mdc_tag = MDC_OPTIONS[self.style] + self._mdcFmt = mdcfmt if mdcfmt else '{reqeustID}' - def _mdcfmtKey(self): + def format(self, record): + # type: (LogRecord) -> str """ - maximum barce match algorithm to find the mdc key - :return: key in brace and key not in brace,such as ({key}, key) + Find MDCs in a log record's extra field. If the key from mdcFmt + doesn't contain MDC, the values will be empty. + + For example: + The MDC dict in a logging record is {'key1':'value1','key2':'value2'}. + The mdcFmt is '{key1} {key3}'. + The output of MDC message is 'key1=value1 key3='. + + Args: + record : an instance of a logged event. + Returns: + str : "colored" text (formatted text). + """ + + mdc_index = self._fmt.find(self._mdc_tag) + if mdc_index == -1: + return self._parent_format(record) + + mdc_format_keys, mdc_format_words = self._mdc_format_key() + + if mdc_format_words is None: + self._fmt = self._replace_mdc_tag_str("") + self._apply_styling() + + return self._parent_format(record) + + res = self._apply_mdc(record, mdc_format_words) + + try: + mdc_string = self._replaceStr(keys=mdc_format_keys).format(**res) + self._fmt = self._replace_mdc_tag_str(mdc_string) + self._apply_styling() + + return self._parent_format(record) + + except KeyError as e: + # is there a need for print? + print("The mdc key %s format is wrong" % str(e)) + + except Exception: + raise + + def _mdc_format_key(self): + # type: () -> (List, Mapping[str, str]) + """Maximum (balanced) parantehses matching algorithm for MDC keys. + + Extracts and strips keys and words from a MDC format string. Use this + method to find the MDC key. + + Returns: + list : list of keys. + map object : keys with and without brace, such as ({key}, key). """ left = '{' right = '}' target = self._mdcFmt - st = [] + stack = [] keys = [] + for index, v in enumerate(target): if v == left: - st.append(index) + stack.append(index) elif v == right: - if len(st) == 0: + if len(stack) == 0: continue - elif len(st) == 1: - start = st.pop() + elif len(stack) == 1: + start = stack.pop() end = index keys.append(target[start:end + 1]) - elif len(st) > 0: - st.pop() + elif len(stack) > 0: + stack.pop() keys = list(filter(lambda x: x[1:-1].strip('\n \t ') != "", keys)) words = None + if keys: words = map(lambda x: x[1:-1], keys) return keys, words - def _replaceStr(self, keys): - - fmt = self._mdcFmt - for i in keys: - fmt = fmt.replace(i, i[1:-1] + "=" + i) - - return fmt - - def format(self, record): + def _replace_string(self, keys): + # type: (List[str]) -> str """ - Find mdcs in log record extra field, if key form mdcFmt dosen't - contains mdcs, the values will be empty. - :param record: the logging record instance - :return: string - for example: - the mdcs dict in logging record is - {'key1':'value1','key2':'value2'} - the mdcFmt is" '{key1} {key3}' - the output of mdc message: 'key1=value1 key3=' - + Removes the first and last characters from each key and assigns not + stripped keys. """ - mdcIndex = self._fmt.find(self._mdc_tag) - if mdcIndex == -1: - if is_above_python_2_7(): - return super(MDCFormatter, self).format(record) - else: - return MarkerFormatter.format(self, record) - - mdcFmtkeys, mdcFmtWords = self._mdcfmtKey() - - if mdcFmtWords is None: - self._fmt = self._fmt.replace(self._mdc_tag, "") - if is_above_python_3_2(): - self._style = logging._STYLES[self.style][0](self._fmt) + fmt = self._mdcFmt + for key in keys: + fmt = fmt.replace(key, key[1:-1] + "=" + key) + return fmt - if is_above_python_2_7(): - return super(MDCFormatter, self).format(record) - else: - return MarkerFormatter.format(self, record) + def _parent_format(self, record): + # type: (LogRecord) -> str + """Call super class's format based on Python version.""" + if is_above_python_2_7(): + return super(MDCFormatter, self).format(record) + else: + return MarkerFormatter.format(self, record) + def _apply_mdc(self, record, mdc_format_words): + # type: (LogRecord, Mapping[Callable[[str], str], List]) -> Dict + """Apply MDC pamming to the LogRecord record.""" mdc = record.__dict__.get('mdc', None) res = {} - for i in mdcFmtWords: + + for i in mdc_format_words: if mdc and i in mdc: res[i] = mdc[i] else: res[i] = "" - del mdc - try: - mdcstr = self._replaceStr(keys=mdcFmtkeys).format(**res) - self._fmt = self._fmt.replace(self._mdc_tag, mdcstr) + return res - if is_above_python_3_2(): - self._style = logging._STYLES[self.style][0](self._fmt) + def _apply_styling(self): + # type: () -> None + """Apply styling to the formatter if using Python 3.2+""" + if is_above_python_3_2(): + StylingClass = logging._STYLES[self.style][0](self._fmt) + self._style = StylingClass - if is_above_python_2_7(): - return super(MDCFormatter, self).format(record) - else: - return MarkerFormatter.format(self, record) + def _replace_mdc_tag_str(self, replacement): + # type: (str) -> str + """Replace MDC tag in the format string.""" + return self._fmt.replace(self._mdc_tag, replacement) - except KeyError as e: - print("The mdc key %s format is wrong" % str(e)) - except Exception: - raise + @deprecated(reason="Will be replaced. Use _mdc_format_key() instead.") + def _mdcfmtKey(self): + """See _mdc_format_key().""" + return self._mdc_format_key() + + @deprecated(reason="Will be replaced. Use _replace_string(keys) instead.") + def _replaceStr(self, keys): + """See _replace_string(keys).""" + return self._replace_string(keys) diff --git a/pylog/onaplogging/monkey.py b/pylog/onaplogging/monkey.py index f8bf992..33c3ced 100644 --- a/pylog/onaplogging/monkey.py +++ b/pylog/onaplogging/monkey.py @@ -12,17 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .mdcContext import patch_loggingMDC -from .logWatchDog import patch_loggingYaml +from typing import Optional + +from .mdcContext import patch_logging_mdc +from .logWatchDog import patch_logging_yaml __all__ = ["patch_all"] def patch_all(mdc=True, yaml=True): + # type: ( Optional[bool], Optional[bool] ) -> None + """ + Patches both MDC contextual information and YAML configuration file to the + logger by default. To exclude any or both set `mdc` and/or `yaml` + parameters to False. + + Args: + mdc (bool, optional): Defaults to True. + yaml (bool, optional): Defaults to True. + """ if mdc is True: - patch_loggingMDC() + patch_logging_mdc() if yaml is True: - patch_loggingYaml() + patch_logging_yaml() diff --git a/pylog/onaplogging/utils/__init__.py b/pylog/onaplogging/utils/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pylog/onaplogging/utils/__init__.py diff --git a/pylog/onaplogging/utils/styles.py b/pylog/onaplogging/utils/styles.py new file mode 100644 index 0000000..e445aee --- /dev/null +++ b/pylog/onaplogging/utils/styles.py @@ -0,0 +1,84 @@ +# Copyright (c) 2020 Deutsche Telekom. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +"""These are ANSI shell coloring codes used to format strings. + +[ begins the color definition. \033 starts the escape sequence. +[\0330m is the default color of the shell that closes the escape sequence. + +`FMT_STR` takes the color as its first parameter (int). As the second +parameter its takes the text (str). + +TL;DR + Examples on ANSI colors, attributes, backgrounds and foregrounds: + https://stackoverflow.com/a/28938235/7619961 +""" + +COLOR_TAG = "color" +HIGHLIGHT_TAG = "highlight" +ATTRIBUTE_TAG = "attribute" + +RESET = "\033[0m" +FMT_STR = "\033[%dm%s" + +ATTRIBUTES = { + + 'normal': 0, + 'bold': 1, + 'underline': 4, + 'blink': 5, + 'invert': 7, + 'hide': 8, + +} + +HIGHLIGHTS = { + + 'black': 40, + 'red': 41, + 'green': 42, + 'yellow': 43, + 'blue': 44, + 'purple': 45, + 'cyan': 46, + 'white': 47, + +} + +COLORS = { + + 'black': 30, + 'red': 31, + 'green': 32, + 'yellow': 33, + 'blue': 34, + 'purple': 35, + 'cyan': 36, + 'white': 37, + +} + +""" +MDC and MARKER options are used only with Python starting 3.2 due to an update +in the logging module. This allows the use of %-formatting, :meth:`str.format` +(``{}``) formatting or :class:`string.Template` in the format string. +""" + +MARKER_OPTIONS = { + "%": "%(marker)s", + "{": "{marker}", + "$": "${marker}" +} + +MDC_OPTIONS = { + "%": "%(mdc)s", + "{": "{mdc}", + "$": "${mdc}" +} diff --git a/pylog/onaplogging/utils.py b/pylog/onaplogging/utils/system.py index 5c96b2d..cfe49a1 100644 --- a/pylog/onaplogging/utils.py +++ b/pylog/onaplogging/utils/system.py @@ -11,21 +11,21 @@ import sys -def is_above_python_3_2(): # type: () -> bool +def is_above_python_3_2(): + # type: () -> bool """Check if code is running at least on Python 3.2 version. Returns: bool: True if it's at least 3.2 version, False otherwise - """ return sys.version_info >= (3, 2, 0, "final", 0) -def is_above_python_2_7(): # type: () -> bool +def is_above_python_2_7(): + # type: () -> bool """Check if code is running at least on Python 2.7 version. Returns: bool: True if it's at least 2.7 version, False otherwise - """ return sys.version_info >= (2, 7, 0, "final", 0) diff --git a/pylog/onaplogging/utils/tools.py b/pylog/onaplogging/utils/tools.py new file mode 100644 index 0000000..0cb0129 --- /dev/null +++ b/pylog/onaplogging/utils/tools.py @@ -0,0 +1,32 @@ +# Copyright (c) 2020 Deutsche Telekom. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import yaml + +from deprecated import deprecated + + +def yaml_to_dict(filepath): + # type: (str) -> dict + """YAML to Python dict converter. + + Args: + filepath : The filepath to a YAML file. + Returns: + dict : Python dictionary object. + """ + with open(filepath, 'rt') as f: + return yaml.load(f.read()) + + +@deprecated(reason="Will be removed. Call yaml_to_dict(filepath) instead.") +def _yaml2Dict(filename): + """YAML to dict. See yaml_to_dict(filepath).""" + return yaml_to_dict(filename) diff --git a/pylog/requirements.txt b/pylog/requirements.txt index 16afb32..be12b1b 100644 --- a/pylog/requirements.txt +++ b/pylog/requirements.txt @@ -1,2 +1,4 @@ PyYAML watchdog +deprecated +typing diff --git a/pylog/test_requirements.txt b/pylog/test_requirements.txt index d32736b..a5c4199 100644 --- a/pylog/test_requirements.txt +++ b/pylog/test_requirements.txt @@ -1,3 +1,5 @@ -r requirements.txt pytest -mock
\ No newline at end of file +mock +deprecated +typing
\ No newline at end of file diff --git a/pylog/tests/test_color_formatter.py b/pylog/tests/test_color_formatter.py index 9a9ae5a..56bf9c2 100644 --- a/pylog/tests/test_color_formatter.py +++ b/pylog/tests/test_color_formatter.py @@ -27,7 +27,7 @@ from onaplogging.colorFormatter import ( FMT_STR, RESET, ) -from onaplogging.utils import is_above_python_3_2 +from onaplogging.utils.system import is_above_python_3_2 class TestColorFormatter(unittest.TestCase): diff --git a/pylog/tests/test_log_watchdog.py b/pylog/tests/test_log_watchdog.py index 5f43138..e1b9fea 100644 --- a/pylog/tests/test_log_watchdog.py +++ b/pylog/tests/test_log_watchdog.py @@ -22,7 +22,8 @@ if sys.version_info[0] >= 3: import pytest import yaml -from onaplogging.logWatchDog import FileEventHandlers, _yaml2Dict, _yamlConfig +from onaplogging.logWatchDog import FileEventHandlers, _yamlConfig +from onaplogging.utils.tools import yaml_to_dict, _yaml2Dict TestEvent = namedtuple("TestEvent", ["src_path"]) diff --git a/pylog/tests/test_marker_formatter.py b/pylog/tests/test_marker_formatter.py index bda7f24..c7ae6b1 100644 --- a/pylog/tests/test_marker_formatter.py +++ b/pylog/tests/test_marker_formatter.py @@ -30,16 +30,16 @@ class TestMarkerFormatter(unittest.TestCase): def test_marker_formatter_init(self): marker_formatter = MarkerFormatter() self.assertEqual(marker_formatter.style, "%") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") if sys.version_info[0] >= 3: marker_formatter = MarkerFormatter(style="{") self.assertEqual(marker_formatter.style, "{") - self.assertEqual(marker_formatter._marker_tag, "{marker}") + self.assertEqual(marker_formatter.marker_tag, "{marker}") marker_formatter = MarkerFormatter(style="$") self.assertEqual(marker_formatter.style, "$") - self.assertEqual(marker_formatter._marker_tag, "${marker}") + self.assertEqual(marker_formatter.marker_tag, "${marker}") with pytest.raises(ValueError): MarkerFormatter(style="*") @@ -50,18 +50,18 @@ class TestMarkerFormatter(unittest.TestCase): with patch("onaplogging.markerFormatter.BaseColorFormatter.format") as mock_format: marker_formatter = MarkerFormatter() self.assertEqual(marker_formatter._fmt, "%(message)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") marker_formatter.format(record) mock_format.assert_called_once() self.assertEqual(marker_formatter._fmt, "%(message)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") with patch("onaplogging.markerFormatter.BaseColorFormatter.format") as mock_format: marker_formatter = MarkerFormatter(fmt="%(message)s %(marker)s") self.assertEqual(marker_formatter._fmt, "%(message)s %(marker)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") marker_formatter.format(record) mock_format.assert_called_once() self.assertEqual(marker_formatter._fmt, "%(message)s %(marker)s") - self.assertEqual(marker_formatter._marker_tag, "%(marker)s") - self.assertEqual(marker_formatter._tmpFmt, "%(message)s %(marker)s") + self.assertEqual(marker_formatter.marker_tag, "%(marker)s") + self.assertEqual(marker_formatter.temp_fmt, "%(message)s %(marker)s") diff --git a/pylog/tests/test_monkey.py b/pylog/tests/test_monkey.py index 4f71fe2..9b64b62 100644 --- a/pylog/tests/test_monkey.py +++ b/pylog/tests/test_monkey.py @@ -16,32 +16,32 @@ if sys.version_info[0] < 3: if sys.version_info[0] >= 3: from unittest.mock import patch -from onaplogging.monkey import patch_all, patch_loggingMDC, patch_loggingYaml +from onaplogging.monkey import patch_all, patch_logging_yaml, patch_logging_mdc class TestMonkey(unittest.TestCase): def test_patch_all(self): - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all() mock_mdc.assert_called_once() mock_yaml.assert_called_once() - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all(mdc=False) mock_mdc.assert_not_called() mock_yaml.assert_called_once() - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all(yaml=False) mock_mdc.assert_called_once() mock_yaml.assert_not_called() - with patch("onaplogging.monkey.patch_loggingMDC") as mock_mdc: - with patch("onaplogging.monkey.patch_loggingYaml") as mock_yaml: + with patch("onaplogging.monkey.patch_logging_mdc") as mock_mdc: + with patch("onaplogging.monkey.patch_logging_yaml") as mock_yaml: patch_all(mdc=False, yaml=False) mock_mdc.assert_not_called() mock_yaml.assert_not_called() diff --git a/pylog/tests/test_utils.py b/pylog/tests/test_utils.py index 5a64aab..a118361 100644 --- a/pylog/tests/test_utils.py +++ b/pylog/tests/test_utils.py @@ -16,30 +16,30 @@ if sys.version_info[0] < 3: if sys.version_info[0] >= 3: from unittest.mock import patch, MagicMock -from onaplogging.utils import is_above_python_2_7, is_above_python_3_2 +from onaplogging.utils.system import is_above_python_2_7, is_above_python_3_2 class TestUtils(unittest.TestCase): def test_is_above_python_3_2(self): - with patch("onaplogging.utils.sys.version_info", (3, 4, 7)): + with patch("onaplogging.utils.system.sys.version_info", (3, 4, 7)): assert is_above_python_3_2() is True - with patch("onaplogging.utils.sys.version_info", (2, 7, 5)): + with patch("onaplogging.utils.system.sys.version_info", (2, 7, 5)): assert is_above_python_3_2() is False - with patch("onaplogging.utils.sys.version_info", (3, 2, 0, "final", 0)): + with patch("onaplogging.utils.system.sys.version_info", (3, 2, 0, "final", 0)): assert is_above_python_3_2() is True def test_is_above_python_2_7(self): - with patch("onaplogging.utils.sys.version_info", (3, 4, 7)): + with patch("onaplogging.utils.system.sys.version_info", (3, 4, 7)): assert is_above_python_2_7() is True - with patch("onaplogging.utils.sys.version_info", (2, 7, 5)): + with patch("onaplogging.utils.system.sys.version_info", (2, 7, 5)): assert is_above_python_2_7() is True - with patch("onaplogging.utils.sys.version_info", (2, 5, 6)): + with patch("onaplogging.utils.system.sys.version_info", (2, 5, 6)): assert is_above_python_2_7() is False - with patch("onaplogging.utils.sys.version_info", (2, 7, 0, "final", 0)): + with patch("onaplogging.utils.system.sys.version_info", (2, 7, 0, "final", 0)): assert is_above_python_2_7() is True diff --git a/pylog/tox.ini b/pylog/tox.ini index b5f2dd3..355385c 100644 --- a/pylog/tox.ini +++ b/pylog/tox.ini @@ -1,5 +1,6 @@ [tox] envlist = + py27 py36 py37 py38 @@ -18,6 +19,9 @@ deps = -r{toxinidir}/test_requirements.txt setenv = PYTHONPATH={toxinidir}/ commands = pytest +[flake8] +ignore = E271, E125, E128, E127 + [testenv:pep8] basepython = python3 deps=flake8 |