From 15fc9df448221c4d24fe4c097fe5e00b4512f083 Mon Sep 17 00:00:00 2001 From: liangke Date: Wed, 28 Feb 2018 15:22:43 +0800 Subject: Submit python logging library seed code Change-Id: I4c039a667d7b8c7a257b2d50f94370785100a968 Issue-ID: MULTICLOUD-151 Issue-ID: LOG-161 Signed-off-by: liangke --- pylog/onaplogging/__init__.py | 11 +++ pylog/onaplogging/logWatchDog.py | 95 ++++++++++++++++++++++ pylog/onaplogging/mdcContext.py | 166 ++++++++++++++++++++++++++++++++++++++ pylog/onaplogging/mdcformatter.py | 123 ++++++++++++++++++++++++++++ pylog/onaplogging/monkey.py | 27 +++++++ 5 files changed, 422 insertions(+) create mode 100644 pylog/onaplogging/__init__.py create mode 100644 pylog/onaplogging/logWatchDog.py create mode 100644 pylog/onaplogging/mdcContext.py create mode 100644 pylog/onaplogging/mdcformatter.py create mode 100644 pylog/onaplogging/monkey.py (limited to 'pylog/onaplogging') diff --git a/pylog/onaplogging/__init__.py b/pylog/onaplogging/__init__.py new file mode 100644 index 0000000..1f2f9d9 --- /dev/null +++ b/pylog/onaplogging/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pylog/onaplogging/logWatchDog.py b/pylog/onaplogging/logWatchDog.py new file mode 100644 index 0000000..e0673e3 --- /dev/null +++ b/pylog/onaplogging/logWatchDog.py @@ -0,0 +1,95 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import os +import yaml +import traceback +from logging import config +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + + +__all__ = ['patch_loggingYaml'] + + +def _yaml2Dict(filename): + + with open(filename, 'rt') as f: + return yaml.load(f.read()) + + +class FileEventHandlers(FileSystemEventHandler): + + def __init__(self, filepath): + + FileSystemEventHandler.__init__(self) + self.filepath = filepath + self.currentConfig = None + + def on_modified(self, event): + try: + if event.src_path == self.filepath: + newConfig = _yaml2Dict(self.filepath) + print ("reload logging configure file %s" % event.src_path) + config.dictConfig(newConfig) + self.currentConfig = newConfig + + except Exception as e: + traceback.print_exc(e) + print ("Reuse the old configuration to avoid this " + "exception terminate program") + if self.currentConfig: + config.dictConfig(self.currentConfig) + + +def _yamlConfig(filepath=None, watchDog=None): + + """ + load logging configureation from yaml file and monitor file status + + :param filepath: logging yaml configure file absolute path + :param watchDog: monitor yaml file identifier status + :return: + """ + if os.path.isfile(filepath) is False: + raise OSError("wrong file") + + dirpath = os.path.dirname(filepath) + event_handler = None + + try: + dictConfig = _yaml2Dict(filepath) + # The watchdog could monitor yaml file status,if be modified + # will send a notify then we could reload logging configuration + if watchDog: + observer = Observer() + event_handler = FileEventHandlers(filepath) + observer.schedule(event_handler=event_handler, path=dirpath, + recursive=False) + observer.setDaemon(True) + observer.start() + + config.dictConfig(dictConfig) + + if event_handler: + # here we keep the correct configuration for reusing + event_handler.currentConfig = dictConfig + + except Exception as e: + traceback.print_exc(e) + + +def patch_loggingYaml(): + + # The patch to add yam config forlogginf and runtime + # reload logging when modify yaml file + config.yamlConfig = _yamlConfig diff --git a/pylog/onaplogging/mdcContext.py b/pylog/onaplogging/mdcContext.py new file mode 100644 index 0000000..8162b50 --- /dev/null +++ b/pylog/onaplogging/mdcContext.py @@ -0,0 +1,166 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +import logging +import threading +import os +import sys +import functools + + +__all__ = ['patch_loggingMDC', 'MDC'] + +_replace_func_name = ['info', 'critical', 'fatal', 'debug', + 'error', 'warn', 'warning', 'findCaller'] + + +class MDCContext(threading.local): + """ + A Thread local instance to storage mdc values + """ + def __init__(self): + + super(MDCContext, self).__init__() + self._localDict = {} + + def get(self, key): + + return self._localDict.get(key, None) + + def put(self, key, value): + + self._localDict[key] = value + + def remove(self, key): + + if key in self.localDict: + del self._localDict[key] + + def clear(self): + + self._localDict.clear() + + def result(self): + + return self._localDict + + def isEmpty(self): + + return self._localDict == {} or self._localDict is None + + +MDC = MDCContext() + + +def fetchkeys(func): + + @functools.wraps(func) + def replace(*args, **kwargs): + kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None)) + func(*args, **kwargs) + return replace + + +def _getmdcs(extra=None): + """ + Put mdc dict in logging record extra filed with key 'mdc' + :param extra: dict + :return: mdc dict + """ + if MDC.isEmpty(): + return + + mdc = MDC.result() + + if extra is not None: + for key in extra: + # make sure extra key dosen't override mdckey + if key in mdc or key == 'mdc': + raise KeyError("Attempt to overwrite %r in MDC" % key) + else: + extra = {} + + extra['mdc'] = mdc + del mdc + return extra + + +@fetchkeys +def info(self, msg, *args, **kwargs): + + if self.isEnabledFor(logging.INFO): + self._log(logging.INFO, msg, args, **kwargs) + + +@fetchkeys +def debug(self, msg, *args, **kwargs): + + if self.isEnabledFor(logging.DEBUG): + self._log(logging.DEBUG, msg, args, **kwargs) + + +@fetchkeys +def warning(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.WARNING): + self._log(logging.WARNING, msg, args, **kwargs) + + +@fetchkeys +def exception(self, msg, *args, **kwargs): + + kwargs['exc_info'] = 1 + self.error(msg, *args, **kwargs) + + +@fetchkeys +def critical(self, msg, *args, **kwargs): + + if self.isEnabledFor(logging.CRITICAL): + self._log(logging.CRITICAL, msg, args, **kwargs) + + +@fetchkeys +def error(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.ERROR): + self._log(logging.ERROR, msg, args, **kwargs) + + +def findCaller(self): + + f = logging.currentframe() + if f is not None: + f = f.f_back + rv = "(unkown file)", 0, "(unknow function)" + while hasattr(f, "f_code"): + co = f.f_code + filename = os.path.normcase(co.co_filename) + # jump through local 'replace' func frame + if filename == logging._srcfile or co.co_name == "replace": + f = f.f_back + continue + rv = (co.co_filename, f.f_lineno, co.co_name) + break + + return rv + + +def patch_loggingMDC(): + """ + The patch to add MDC ability in logging Record instance at runtime + """ + localModule = sys.modules[__name__] + for attr in dir(logging.Logger): + if attr in _replace_func_name: + newfunc = getattr(localModule, attr, None) + if newfunc: + setattr(logging.Logger, attr, newfunc) diff --git a/pylog/onaplogging/mdcformatter.py b/pylog/onaplogging/mdcformatter.py new file mode 100644 index 0000000..f63ec94 --- /dev/null +++ b/pylog/onaplogging/mdcformatter.py @@ -0,0 +1,123 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import logging + + +class MDCFormatter(logging.Formatter): + """ + A custom MDC formatter to prepare Mapped Diagnostic Context + to enrich log message. + """ + + def __init__(self, fmt=None, mdcfmt=None, datefmt=None): + """ + :param fmt: build-in format string contains standard + Python %-style mapping keys + :param mdcFmt: mdc format with '{}'-style mapping keys + :param datefmt: Date format to use + """ + + super(MDCFormatter, self).__init__(fmt=fmt, datefmt=datefmt) + self._tmpfmt = self._fmt + if mdcfmt: + self._mdcFmt = mdcfmt + else: + self._mdcFmt = '{reqeustID}' + + def _mdcfmtKey(self): + """ + maximum barce match algorithm to find the mdc key + :return: key in brace and key not in brace,such as ({key}, key) + """ + + left = '{' + right = '}' + target = self._mdcFmt + st = [] + keys = [] + for index, v in enumerate(target): + if v == left: + st.append(index) + elif v == right: + + if len(st) == 0: + continue + + elif len(st) == 1: + start = st.pop() + end = index + keys.append(target[start:end + 1]) + elif len(st) > 0: + st.pop() + + keys = filter(lambda x: x[1:-1].strip('\n \t ') != "", keys) + words = None + if keys: + words = map(lambda x: x[1:-1], keys) + + return keys, words + + def _replaceStr(self, keys): + + fmt = self._mdcFmt + for i in keys: + fmt = fmt.replace(i, i[1:-1] + "=" + i) + + return fmt + + def format(self, record): + """ + Find mdcs in log record extra field, if key form mdcFmt dosen't + contains mdcs, the values will be empty. + :param record: the logging record instance + :return: string + for example: + the mdcs dict in logging record is + {'key1':'value1','key2':'value2'} + the mdcFmt is" '{key1} {key3}' + the output of mdc message: 'key1=value1 key3=' + + """ + mdcIndex = self._fmt.find('%(mdc)s') + if mdcIndex == -1: + return super(MDCFormatter, self).format(record) + + mdcFmtkeys, mdcFmtWords = self._mdcfmtKey() + if mdcFmtWords is None: + + self._fmt = self._fmt.replace("%(mdc)s", "") + return super(MDCFormatter, self).format(record) + + mdc = record.__dict__.get('mdc', None) + res = {} + for i in mdcFmtWords: + if mdc and i in mdc: + res[i] = mdc[i] + else: + res[i] = "" + + del mdc + try: + mdcstr = self._replaceStr(keys=mdcFmtkeys).format(**res) + self._fmt = self._fmt.replace("%(mdc)s", mdcstr) + s = super(MDCFormatter, self).format(record) + return s + + except KeyError as e: + print ("The mdc key %s format is wrong" % e.message) + except Exception: + raise + + finally: + # reset fmt format + self._fmt = self._tmpfmt diff --git a/pylog/onaplogging/monkey.py b/pylog/onaplogging/monkey.py new file mode 100644 index 0000000..fcf8fdf --- /dev/null +++ b/pylog/onaplogging/monkey.py @@ -0,0 +1,27 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +from mdcContext import patch_loggingMDC +from logWatchDog import patch_loggingYaml + + +__all__ = ["patch_all"] + + +def patch_all(mdc=True, yaml=True): + + if mdc is True: + patch_loggingMDC() + + if yaml is True: + patch_loggingYaml() -- cgit 1.2.3-korg