summaryrefslogtreecommitdiffstats
path: root/pylog/onaplogging
diff options
context:
space:
mode:
authorliangke <lokyse@163.com>2018-02-28 15:22:43 +0800
committerliangke <lokyse@163.com>2018-03-01 12:52:58 +0800
commit15fc9df448221c4d24fe4c097fe5e00b4512f083 (patch)
treef105b65035c8e576249d5e1d781253b181f35862 /pylog/onaplogging
parent9379dc6c5bd516a7601a179603de1aeefc5140ec (diff)
Submit python logging library seed code
Change-Id: I4c039a667d7b8c7a257b2d50f94370785100a968 Issue-ID: MULTICLOUD-151 Issue-ID: LOG-161 Signed-off-by: liangke <lokyse@163.com>
Diffstat (limited to 'pylog/onaplogging')
-rw-r--r--pylog/onaplogging/__init__.py11
-rw-r--r--pylog/onaplogging/logWatchDog.py95
-rw-r--r--pylog/onaplogging/mdcContext.py166
-rw-r--r--pylog/onaplogging/mdcformatter.py123
-rw-r--r--pylog/onaplogging/monkey.py27
5 files changed, 422 insertions, 0 deletions
diff --git a/pylog/onaplogging/__init__.py b/pylog/onaplogging/__init__.py
new file mode 100644
index 0000000..1f2f9d9
--- /dev/null
+++ b/pylog/onaplogging/__init__.py
@@ -0,0 +1,11 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/pylog/onaplogging/logWatchDog.py b/pylog/onaplogging/logWatchDog.py
new file mode 100644
index 0000000..e0673e3
--- /dev/null
+++ b/pylog/onaplogging/logWatchDog.py
@@ -0,0 +1,95 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import os
+import yaml
+import traceback
+from logging import config
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+
+__all__ = ['patch_loggingYaml']
+
+
+def _yaml2Dict(filename):
+
+ with open(filename, 'rt') as f:
+ return yaml.load(f.read())
+
+
+class FileEventHandlers(FileSystemEventHandler):
+
+ def __init__(self, filepath):
+
+ FileSystemEventHandler.__init__(self)
+ self.filepath = filepath
+ self.currentConfig = None
+
+ def on_modified(self, event):
+ try:
+ if event.src_path == self.filepath:
+ newConfig = _yaml2Dict(self.filepath)
+ print ("reload logging configure file %s" % event.src_path)
+ config.dictConfig(newConfig)
+ self.currentConfig = newConfig
+
+ except Exception as e:
+ traceback.print_exc(e)
+ print ("Reuse the old configuration to avoid this "
+ "exception terminate program")
+ if self.currentConfig:
+ config.dictConfig(self.currentConfig)
+
+
+def _yamlConfig(filepath=None, watchDog=None):
+
+ """
+ load logging configureation from yaml file and monitor file status
+
+ :param filepath: logging yaml configure file absolute path
+ :param watchDog: monitor yaml file identifier status
+ :return:
+ """
+ if os.path.isfile(filepath) is False:
+ raise OSError("wrong file")
+
+ dirpath = os.path.dirname(filepath)
+ event_handler = None
+
+ try:
+ dictConfig = _yaml2Dict(filepath)
+ # The watchdog could monitor yaml file status,if be modified
+ # will send a notify then we could reload logging configuration
+ if watchDog:
+ observer = Observer()
+ event_handler = FileEventHandlers(filepath)
+ observer.schedule(event_handler=event_handler, path=dirpath,
+ recursive=False)
+ observer.setDaemon(True)
+ observer.start()
+
+ config.dictConfig(dictConfig)
+
+ if event_handler:
+ # here we keep the correct configuration for reusing
+ event_handler.currentConfig = dictConfig
+
+ except Exception as e:
+ traceback.print_exc(e)
+
+
+def patch_loggingYaml():
+
+ # The patch to add yam config forlogginf and runtime
+ # reload logging when modify yaml file
+ config.yamlConfig = _yamlConfig
diff --git a/pylog/onaplogging/mdcContext.py b/pylog/onaplogging/mdcContext.py
new file mode 100644
index 0000000..8162b50
--- /dev/null
+++ b/pylog/onaplogging/mdcContext.py
@@ -0,0 +1,166 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+import logging
+import threading
+import os
+import sys
+import functools
+
+
+__all__ = ['patch_loggingMDC', 'MDC']
+
+_replace_func_name = ['info', 'critical', 'fatal', 'debug',
+ 'error', 'warn', 'warning', 'findCaller']
+
+
+class MDCContext(threading.local):
+ """
+ A Thread local instance to storage mdc values
+ """
+ def __init__(self):
+
+ super(MDCContext, self).__init__()
+ self._localDict = {}
+
+ def get(self, key):
+
+ return self._localDict.get(key, None)
+
+ def put(self, key, value):
+
+ self._localDict[key] = value
+
+ def remove(self, key):
+
+ if key in self.localDict:
+ del self._localDict[key]
+
+ def clear(self):
+
+ self._localDict.clear()
+
+ def result(self):
+
+ return self._localDict
+
+ def isEmpty(self):
+
+ return self._localDict == {} or self._localDict is None
+
+
+MDC = MDCContext()
+
+
+def fetchkeys(func):
+
+ @functools.wraps(func)
+ def replace(*args, **kwargs):
+ kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None))
+ func(*args, **kwargs)
+ return replace
+
+
+def _getmdcs(extra=None):
+ """
+ Put mdc dict in logging record extra filed with key 'mdc'
+ :param extra: dict
+ :return: mdc dict
+ """
+ if MDC.isEmpty():
+ return
+
+ mdc = MDC.result()
+
+ if extra is not None:
+ for key in extra:
+ # make sure extra key dosen't override mdckey
+ if key in mdc or key == 'mdc':
+ raise KeyError("Attempt to overwrite %r in MDC" % key)
+ else:
+ extra = {}
+
+ extra['mdc'] = mdc
+ del mdc
+ return extra
+
+
+@fetchkeys
+def info(self, msg, *args, **kwargs):
+
+ if self.isEnabledFor(logging.INFO):
+ self._log(logging.INFO, msg, args, **kwargs)
+
+
+@fetchkeys
+def debug(self, msg, *args, **kwargs):
+
+ if self.isEnabledFor(logging.DEBUG):
+ self._log(logging.DEBUG, msg, args, **kwargs)
+
+
+@fetchkeys
+def warning(self, msg, *args, **kwargs):
+ if self.isEnabledFor(logging.WARNING):
+ self._log(logging.WARNING, msg, args, **kwargs)
+
+
+@fetchkeys
+def exception(self, msg, *args, **kwargs):
+
+ kwargs['exc_info'] = 1
+ self.error(msg, *args, **kwargs)
+
+
+@fetchkeys
+def critical(self, msg, *args, **kwargs):
+
+ if self.isEnabledFor(logging.CRITICAL):
+ self._log(logging.CRITICAL, msg, args, **kwargs)
+
+
+@fetchkeys
+def error(self, msg, *args, **kwargs):
+ if self.isEnabledFor(logging.ERROR):
+ self._log(logging.ERROR, msg, args, **kwargs)
+
+
+def findCaller(self):
+
+ f = logging.currentframe()
+ if f is not None:
+ f = f.f_back
+ rv = "(unkown file)", 0, "(unknow function)"
+ while hasattr(f, "f_code"):
+ co = f.f_code
+ filename = os.path.normcase(co.co_filename)
+ # jump through local 'replace' func frame
+ if filename == logging._srcfile or co.co_name == "replace":
+ f = f.f_back
+ continue
+ rv = (co.co_filename, f.f_lineno, co.co_name)
+ break
+
+ return rv
+
+
+def patch_loggingMDC():
+ """
+ The patch to add MDC ability in logging Record instance at runtime
+ """
+ localModule = sys.modules[__name__]
+ for attr in dir(logging.Logger):
+ if attr in _replace_func_name:
+ newfunc = getattr(localModule, attr, None)
+ if newfunc:
+ setattr(logging.Logger, attr, newfunc)
diff --git a/pylog/onaplogging/mdcformatter.py b/pylog/onaplogging/mdcformatter.py
new file mode 100644
index 0000000..f63ec94
--- /dev/null
+++ b/pylog/onaplogging/mdcformatter.py
@@ -0,0 +1,123 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import logging
+
+
+class MDCFormatter(logging.Formatter):
+ """
+ A custom MDC formatter to prepare Mapped Diagnostic Context
+ to enrich log message.
+ """
+
+ def __init__(self, fmt=None, mdcfmt=None, datefmt=None):
+ """
+ :param fmt: build-in format string contains standard
+ Python %-style mapping keys
+ :param mdcFmt: mdc format with '{}'-style mapping keys
+ :param datefmt: Date format to use
+ """
+
+ super(MDCFormatter, self).__init__(fmt=fmt, datefmt=datefmt)
+ self._tmpfmt = self._fmt
+ if mdcfmt:
+ self._mdcFmt = mdcfmt
+ else:
+ self._mdcFmt = '{reqeustID}'
+
+ def _mdcfmtKey(self):
+ """
+ maximum barce match algorithm to find the mdc key
+ :return: key in brace and key not in brace,such as ({key}, key)
+ """
+
+ left = '{'
+ right = '}'
+ target = self._mdcFmt
+ st = []
+ keys = []
+ for index, v in enumerate(target):
+ if v == left:
+ st.append(index)
+ elif v == right:
+
+ if len(st) == 0:
+ continue
+
+ elif len(st) == 1:
+ start = st.pop()
+ end = index
+ keys.append(target[start:end + 1])
+ elif len(st) > 0:
+ st.pop()
+
+ keys = filter(lambda x: x[1:-1].strip('\n \t ') != "", keys)
+ words = None
+ if keys:
+ words = map(lambda x: x[1:-1], keys)
+
+ return keys, words
+
+ def _replaceStr(self, keys):
+
+ fmt = self._mdcFmt
+ for i in keys:
+ fmt = fmt.replace(i, i[1:-1] + "=" + i)
+
+ return fmt
+
+ def format(self, record):
+ """
+ Find mdcs in log record extra field, if key form mdcFmt dosen't
+ contains mdcs, the values will be empty.
+ :param record: the logging record instance
+ :return: string
+ for example:
+ the mdcs dict in logging record is
+ {'key1':'value1','key2':'value2'}
+ the mdcFmt is" '{key1} {key3}'
+ the output of mdc message: 'key1=value1 key3='
+
+ """
+ mdcIndex = self._fmt.find('%(mdc)s')
+ if mdcIndex == -1:
+ return super(MDCFormatter, self).format(record)
+
+ mdcFmtkeys, mdcFmtWords = self._mdcfmtKey()
+ if mdcFmtWords is None:
+
+ self._fmt = self._fmt.replace("%(mdc)s", "")
+ return super(MDCFormatter, self).format(record)
+
+ mdc = record.__dict__.get('mdc', None)
+ res = {}
+ for i in mdcFmtWords:
+ if mdc and i in mdc:
+ res[i] = mdc[i]
+ else:
+ res[i] = ""
+
+ del mdc
+ try:
+ mdcstr = self._replaceStr(keys=mdcFmtkeys).format(**res)
+ self._fmt = self._fmt.replace("%(mdc)s", mdcstr)
+ s = super(MDCFormatter, self).format(record)
+ return s
+
+ except KeyError as e:
+ print ("The mdc key %s format is wrong" % e.message)
+ except Exception:
+ raise
+
+ finally:
+ # reset fmt format
+ self._fmt = self._tmpfmt
diff --git a/pylog/onaplogging/monkey.py b/pylog/onaplogging/monkey.py
new file mode 100644
index 0000000..fcf8fdf
--- /dev/null
+++ b/pylog/onaplogging/monkey.py
@@ -0,0 +1,27 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+from mdcContext import patch_loggingMDC
+from logWatchDog import patch_loggingYaml
+
+
+__all__ = ["patch_all"]
+
+
+def patch_all(mdc=True, yaml=True):
+
+ if mdc is True:
+ patch_loggingMDC()
+
+ if yaml is True:
+ patch_loggingYaml()