Submit python logging library seed code

Change-Id: I4c039a667d7b8c7a257b2d50f94370785100a968
Issue-ID: MULTICLOUD-151
Issue-ID: LOG-161
Signed-off-by: liangke <lokyse@163.com>

5 files changed, 422 insertions, 0 deletions

diff --git a/pylog/onaplogging/__init__.py b/pylog/onaplogging/__init__.py
new file mode 100644
index 0000000..1f2f9d9
--- /dev/null
+++ b/pylog/onaplogging/__init__.py
@@ -0,0 +1,11 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/pylog/onaplogging/logWatchDog.py b/pylog/onaplogging/logWatchDog.py
new file mode 100644
index 0000000..e0673e3
--- /dev/null
+++ b/pylog/onaplogging/logWatchDog.py
@@ -0,0 +1,95 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import os
+import yaml
+import traceback
+from logging import config
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+
+__all__ = ['patch_loggingYaml']
+
+
+def _yaml2Dict(filename):
+
+    with open(filename, 'rt') as f:
+        return yaml.load(f.read())
+
+
+class FileEventHandlers(FileSystemEventHandler):
+
+    def __init__(self, filepath):
+
+        FileSystemEventHandler.__init__(self)
+        self.filepath = filepath
+        self.currentConfig = None
+
+    def on_modified(self, event):
+        try:
+            if event.src_path == self.filepath:
+                newConfig = _yaml2Dict(self.filepath)
+                print ("reload logging configure file %s" % event.src_path)
+                config.dictConfig(newConfig)
+                self.currentConfig = newConfig
+
+        except Exception as e:
+            traceback.print_exc(e)
+            print ("Reuse the old configuration to avoid this "
+                   "exception terminate program")
+            if self.currentConfig:
+                config.dictConfig(self.currentConfig)
+
+
+def _yamlConfig(filepath=None, watchDog=None):
+
+    """
+    load logging configureation from yaml file and monitor file status
+
+    :param filepath: logging yaml configure file absolute path
+    :param watchDog: monitor yaml file identifier status
+    :return:
+    """
+    if os.path.isfile(filepath) is False:
+        raise OSError("wrong file")
+
+    dirpath = os.path.dirname(filepath)
+    event_handler = None
+
+    try:
+        dictConfig = _yaml2Dict(filepath)
+        #  The watchdog could monitor yaml file status,if be modified
+        #  will send a notify  then we could reload logging configuration
+        if watchDog:
+            observer = Observer()
+            event_handler = FileEventHandlers(filepath)
+            observer.schedule(event_handler=event_handler, path=dirpath,
+                              recursive=False)
+            observer.setDaemon(True)
+            observer.start()
+
+        config.dictConfig(dictConfig)
+
+        if event_handler:
+            # here we keep the correct configuration for reusing
+            event_handler.currentConfig = dictConfig
+
+    except Exception as e:
+        traceback.print_exc(e)
+
+
+def patch_loggingYaml():
+
+    # The patch to add yam config forlogginf and runtime
+    # reload logging when modify yaml file
+    config.yamlConfig = _yamlConfig
diff --git a/pylog/onaplogging/mdcContext.py b/pylog/onaplogging/mdcContext.py
new file mode 100644
index 0000000..8162b50
--- /dev/null
+++ b/pylog/onaplogging/mdcContext.py
@@ -0,0 +1,166 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+import logging
+import threading
+import os
+import sys
+import functools
+
+
+__all__ = ['patch_loggingMDC', 'MDC']
+
+_replace_func_name = ['info', 'critical', 'fatal', 'debug',
+                      'error', 'warn', 'warning', 'findCaller']
+
+
+class MDCContext(threading.local):
+    """
+    A Thread local instance to storage mdc values
+    """
+    def __init__(self):
+
+        super(MDCContext, self).__init__()
+        self._localDict = {}
+
+    def get(self, key):
+
+        return self._localDict.get(key, None)
+
+    def put(self, key, value):
+
+        self._localDict[key] = value
+
+    def remove(self, key):
+
+        if key in self.localDict:
+            del self._localDict[key]
+
+    def clear(self):
+
+        self._localDict.clear()
+
+    def result(self):
+
+        return self._localDict
+
+    def isEmpty(self):
+
+        return self._localDict == {} or self._localDict is None
+
+
+MDC = MDCContext()
+
+
+def fetchkeys(func):
+
+    @functools.wraps(func)
+    def replace(*args, **kwargs):
+        kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None))
+        func(*args, **kwargs)
+    return replace
+
+
+def _getmdcs(extra=None):
+    """
+    Put mdc dict in logging record extra filed with key 'mdc'
+    :param extra: dict
+    :return: mdc dict
+    """
+    if MDC.isEmpty():
+        return
+
+    mdc = MDC.result()
+
+    if extra is not None:
+        for key in extra:
+            #  make sure extra key dosen't override mdckey
+            if key in mdc or key == 'mdc':
+                    raise KeyError("Attempt to overwrite %r in MDC" % key)
+    else:
+        extra = {}
+
+    extra['mdc'] = mdc
+    del mdc
+    return extra
+
+
+@fetchkeys
+def info(self, msg, *args, **kwargs):
+
+    if self.isEnabledFor(logging.INFO):
+        self._log(logging.INFO, msg, args, **kwargs)
+
+
+@fetchkeys
+def debug(self, msg, *args, **kwargs):
+
+    if self.isEnabledFor(logging.DEBUG):
+        self._log(logging.DEBUG, msg, args, **kwargs)
+
+
+@fetchkeys
+def warning(self, msg, *args, **kwargs):
+    if self.isEnabledFor(logging.WARNING):
+        self._log(logging.WARNING, msg, args, **kwargs)
+
+
+@fetchkeys
+def exception(self, msg, *args, **kwargs):
+
+    kwargs['exc_info'] = 1
+    self.error(msg, *args, **kwargs)
+
+
+@fetchkeys
+def critical(self, msg, *args, **kwargs):
+
+    if self.isEnabledFor(logging.CRITICAL):
+        self._log(logging.CRITICAL, msg, args, **kwargs)
+
+
+@fetchkeys
+def error(self, msg, *args, **kwargs):
+    if self.isEnabledFor(logging.ERROR):
+        self._log(logging.ERROR, msg, args, **kwargs)
+
+
+def findCaller(self):
+
+    f = logging.currentframe()
+    if f is not None:
+        f = f.f_back
+    rv = "(unkown file)", 0, "(unknow function)"
+    while hasattr(f, "f_code"):
+        co = f.f_code
+        filename = os.path.normcase(co.co_filename)
+        # jump through local 'replace' func frame
+        if filename == logging._srcfile or co.co_name == "replace":
+            f = f.f_back
+            continue
+        rv = (co.co_filename, f.f_lineno, co.co_name)
+        break
+
+    return rv
+
+
+def patch_loggingMDC():
+    """
+    The patch to add MDC ability in logging Record instance at runtime
+    """
+    localModule = sys.modules[__name__]
+    for attr in dir(logging.Logger):
+        if attr in _replace_func_name:
+            newfunc = getattr(localModule, attr, None)
+            if newfunc:
+                setattr(logging.Logger, attr, newfunc)
diff --git a/pylog/onaplogging/mdcformatter.py b/pylog/onaplogging/mdcformatter.py
new file mode 100644
index 0000000..f63ec94
--- /dev/null
+++ b/pylog/onaplogging/mdcformatter.py
@@ -0,0 +1,123 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+import logging
+
+
+class MDCFormatter(logging.Formatter):
+    """
+    A custom MDC formatter to prepare Mapped Diagnostic Context
+    to enrich log message.
+    """
+
+    def __init__(self, fmt=None, mdcfmt=None, datefmt=None):
+        """
+        :param fmt: build-in format string contains standard
+               Python %-style mapping keys
+        :param mdcFmt: mdc format with '{}'-style mapping keys
+        :param datefmt: Date format to use
+        """
+
+        super(MDCFormatter, self).__init__(fmt=fmt, datefmt=datefmt)
+        self._tmpfmt = self._fmt
+        if mdcfmt:
+            self._mdcFmt = mdcfmt
+        else:
+            self._mdcFmt = '{reqeustID}'
+
+    def _mdcfmtKey(self):
+        """
+         maximum barce match algorithm to find the mdc key
+        :return: key in brace  and key not in brace,such as ({key}, key)
+        """
+
+        left = '{'
+        right = '}'
+        target = self._mdcFmt
+        st = []
+        keys = []
+        for index, v in enumerate(target):
+            if v == left:
+                st.append(index)
+            elif v == right:
+
+                if len(st) == 0:
+                    continue
+
+                elif len(st) == 1:
+                    start = st.pop()
+                    end = index
+                    keys.append(target[start:end + 1])
+                elif len(st) > 0:
+                    st.pop()
+
+        keys = filter(lambda x: x[1:-1].strip('\n \t  ') != "", keys)
+        words = None
+        if keys:
+            words = map(lambda x: x[1:-1], keys)
+
+        return keys, words
+
+    def _replaceStr(self, keys):
+
+        fmt = self._mdcFmt
+        for i in keys:
+            fmt = fmt.replace(i, i[1:-1] + "=" + i)
+
+        return fmt
+
+    def format(self, record):
+        """
+        Find mdcs in log record extra field, if key form mdcFmt dosen't
+        contains mdcs, the values will be empty.
+        :param record: the logging record instance
+        :return:  string
+        for example:
+            the mdcs dict in logging record is
+            {'key1':'value1','key2':'value2'}
+            the mdcFmt is" '{key1} {key3}'
+            the output of mdc message: 'key1=value1 key3='
+
+        """
+        mdcIndex = self._fmt.find('%(mdc)s')
+        if mdcIndex == -1:
+            return super(MDCFormatter, self).format(record)
+
+        mdcFmtkeys, mdcFmtWords = self._mdcfmtKey()
+        if mdcFmtWords is None:
+
+            self._fmt = self._fmt.replace("%(mdc)s", "")
+            return super(MDCFormatter, self).format(record)
+
+        mdc = record.__dict__.get('mdc', None)
+        res = {}
+        for i in mdcFmtWords:
+            if mdc and i in mdc:
+                res[i] = mdc[i]
+            else:
+                res[i] = ""
+
+        del mdc
+        try:
+            mdcstr = self._replaceStr(keys=mdcFmtkeys).format(**res)
+            self._fmt = self._fmt.replace("%(mdc)s", mdcstr)
+            s = super(MDCFormatter, self).format(record)
+            return s
+
+        except KeyError as e:
+            print ("The mdc key %s format is wrong" % e.message)
+        except Exception:
+            raise
+
+        finally:
+            # reset fmt format
+            self._fmt = self._tmpfmt
diff --git a/pylog/onaplogging/monkey.py b/pylog/onaplogging/monkey.py
new file mode 100644
index 0000000..fcf8fdf
--- /dev/null
+++ b/pylog/onaplogging/monkey.py
@@ -0,0 +1,27 @@
+# Copyright (c) 2018 VMware, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+
+
+from mdcContext import patch_loggingMDC
+from logWatchDog import patch_loggingYaml
+
+
+__all__ = ["patch_all"]
+
+
+def patch_all(mdc=True, yaml=True):
+
+    if mdc is True:
+        patch_loggingMDC()
+
+    if yaml is True:
+        patch_loggingYaml()