diff options
Diffstat (limited to 'pylog')
| -rw-r--r-- | pylog/LICENSE.txt | 11 | ||||
| -rw-r--r-- | pylog/README.md | 118 | ||||
| -rw-r--r-- | pylog/__init__.py | 0 | ||||
| -rw-r--r-- | pylog/assembly.xml | 51 | ||||
| -rw-r--r-- | pylog/onaplogging/__init__.py | 11 | ||||
| -rw-r--r-- | pylog/onaplogging/logWatchDog.py | 95 | ||||
| -rw-r--r-- | pylog/onaplogging/mdcContext.py | 166 | ||||
| -rw-r--r-- | pylog/onaplogging/mdcformatter.py | 123 | ||||
| -rw-r--r-- | pylog/onaplogging/monkey.py | 27 | ||||
| -rw-r--r-- | pylog/pom.xml | 50 | ||||
| -rw-r--r-- | pylog/requirements.txt | 2 | ||||
| -rw-r--r-- | pylog/setup.py | 37 | ||||
| -rw-r--r-- | pylog/tests/__init__.py | 0 | ||||
| -rw-r--r-- | pylog/tests/test_example.py | 18 | ||||
| -rw-r--r-- | pylog/tox.ini | 34 |
15 files changed, 743 insertions, 0 deletions
diff --git a/pylog/LICENSE.txt b/pylog/LICENSE.txt new file mode 100644 index 0000000..1f2f9d9 --- /dev/null +++ b/pylog/LICENSE.txt @@ -0,0 +1,11 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pylog/README.md b/pylog/README.md new file mode 100644 index 0000000..d221c03 --- /dev/null +++ b/pylog/README.md @@ -0,0 +1,118 @@ +# ONAP python logging package +- python-package onappylog extend python standard logging library which +could be used in any python project to log MDC(Mapped Diagnostic Contex) +and easy to reload logging at runtime. + +----- + +## install package +```bash + pip install onappylog +``` + +## Usage + +### 1. MDC monkey patch + +Import the MDC monkey patch making logRecord to store context in local thread. + +```python + from onaplogging import monkey; monkey.patch_loggingMDC() +``` +Import the MDC format to be used to configure mdc output format. +Please replace your old logging format with mdc format in configuration. + + +```python + from onaplogging import mdcformatter +``` +the mdc format example +```python +'mdcFormater':{ + '()': mdcformatter.MDCFormatter, # Use MDCFormatter instance to convert logging string + 'format': '%(mdc)s and other %-style key ', # Add '%(mdc)s' here. + 'mdcfmt': '{key1} {key2}', # Define your mdc keys here. + 'datefmt': '%Y-%m-%d %H:%M:%S' # date format + } +``` + +Import MDC to store context in python file with logger +code. + +```python +from onaplogging.mdcContext import MDC +# add mdc +MDC.put("key1", "value1") +MDC.put("key2", "value2") + +# origin code +logger.info("msg") +logger.debug("debug") + +``` + +### 2. Reload logging at runtime + +It's thread safe to reload logging. If you want to use this feature, +must use yaml file to configure logging. + + +import the yaml monkey patch and load logging yaml file + +```python + from onaplogging import monkey,monkey.patch_loggingYaml() + # yaml config + config.yamlConfig(filepath=<yaml filepath>, watchDog=True) +``` + +Notice that the watchDog is opening,So your logging could be reloaded at runtime. +if you modify yaml file to change handler、filter or format, +the logger in program will be reloaded to use new configuration. + +Set watchDog to **false**, If you don't need to reloaded logging. + + + + +Yaml configure exmaple + +```yaml +version: 1 + +disable_existing_loggers: True + +loggers: +vio: + level: DEBUG + handlers: [vioHandler] + propagate: False +handlers: +vioHandler: + class: logging.handlers.RotatingFileHandler + level: DEBUG + filename: /var/log/bt.log + mode: a + maxBytes: 1024*1024*50 + backupCount: 10 + formatter: mdcFormatter +formatters: + mdcFormatter: + format: "%(asctime)s:[%(name)s] %(created)f %(module)s %(funcName)s %(pathname)s %(process)d %(levelno)s :[ %(threadName)s %(thread)d]: [%(mdc)s]: [%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" + mdcfmt: "{key1} {key2} {key3} {key4} dwdawdwa " + datefmt: "%Y-%m-%d %H:%M:%S" + (): onaplogging.mdcformatter.MDCFormatter + standard: + format: '%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] + [%(levelname)s]:%(message)s ' + datefmt: "%Y-%m-%d %H:%M:%S" + +``` + + +### 3. reference + +[What's MDC?](https://logging.apache.org/log4j/2.x/manual/thread-context.html) + +[Onap Logging Guidelines](https://wiki.onap.org/pages/viewpage.action?pageId=20087036) + +[Python Standard Logging Library](https://docs.python.org/2/library/logging.html) diff --git a/pylog/__init__.py b/pylog/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pylog/__init__.py diff --git a/pylog/assembly.xml b/pylog/assembly.xml new file mode 100644 index 0000000..3f0d846 --- /dev/null +++ b/pylog/assembly.xml @@ -0,0 +1,51 @@ +<!-- + Copyright (c) 2017-2018 VMware, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +--> +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> + <id>pylog</id> + <formats> + <format>zip</format> + </formats> + <fileSets> + <fileSet> + <directory>onaplogging</directory> + <outputDirectory>/onaplogging</outputDirectory> + <includes> + <include>**/*.py</include> + <include>**/*.json</include> + <include>**/*.xml</include> + </includes> + </fileSet> + <fileSet> + <directory>tests</directory> + <outputDirectory>/tests</outputDirectory> + <includes> + <include>**/*.py</include> + </includes> + </fileSet> + <fileSet> + <directory>.</directory> + <outputDirectory>/</outputDirectory> + <includes> + <include>*.py</include> + <include>*.txt</include> + <include>*.sh</include> + <include>*.ini</include> + <include>*.md</include> + </includes> + </fileSet> + </fileSets> + <baseDirectory>pylog</baseDirectory> + <!--baseDirectory>multivimdriver-broker/multivimbroker</baseDirectory--> +</assembly> diff --git a/pylog/onaplogging/__init__.py b/pylog/onaplogging/__init__.py new file mode 100644 index 0000000..1f2f9d9 --- /dev/null +++ b/pylog/onaplogging/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/pylog/onaplogging/logWatchDog.py b/pylog/onaplogging/logWatchDog.py new file mode 100644 index 0000000..e0673e3 --- /dev/null +++ b/pylog/onaplogging/logWatchDog.py @@ -0,0 +1,95 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import os +import yaml +import traceback +from logging import config +from watchdog.observers import Observer +from watchdog.events import FileSystemEventHandler + + +__all__ = ['patch_loggingYaml'] + + +def _yaml2Dict(filename): + + with open(filename, 'rt') as f: + return yaml.load(f.read()) + + +class FileEventHandlers(FileSystemEventHandler): + + def __init__(self, filepath): + + FileSystemEventHandler.__init__(self) + self.filepath = filepath + self.currentConfig = None + + def on_modified(self, event): + try: + if event.src_path == self.filepath: + newConfig = _yaml2Dict(self.filepath) + print ("reload logging configure file %s" % event.src_path) + config.dictConfig(newConfig) + self.currentConfig = newConfig + + except Exception as e: + traceback.print_exc(e) + print ("Reuse the old configuration to avoid this " + "exception terminate program") + if self.currentConfig: + config.dictConfig(self.currentConfig) + + +def _yamlConfig(filepath=None, watchDog=None): + + """ + load logging configureation from yaml file and monitor file status + + :param filepath: logging yaml configure file absolute path + :param watchDog: monitor yaml file identifier status + :return: + """ + if os.path.isfile(filepath) is False: + raise OSError("wrong file") + + dirpath = os.path.dirname(filepath) + event_handler = None + + try: + dictConfig = _yaml2Dict(filepath) + # The watchdog could monitor yaml file status,if be modified + # will send a notify then we could reload logging configuration + if watchDog: + observer = Observer() + event_handler = FileEventHandlers(filepath) + observer.schedule(event_handler=event_handler, path=dirpath, + recursive=False) + observer.setDaemon(True) + observer.start() + + config.dictConfig(dictConfig) + + if event_handler: + # here we keep the correct configuration for reusing + event_handler.currentConfig = dictConfig + + except Exception as e: + traceback.print_exc(e) + + +def patch_loggingYaml(): + + # The patch to add yam config forlogginf and runtime + # reload logging when modify yaml file + config.yamlConfig = _yamlConfig diff --git a/pylog/onaplogging/mdcContext.py b/pylog/onaplogging/mdcContext.py new file mode 100644 index 0000000..8162b50 --- /dev/null +++ b/pylog/onaplogging/mdcContext.py @@ -0,0 +1,166 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +import logging +import threading +import os +import sys +import functools + + +__all__ = ['patch_loggingMDC', 'MDC'] + +_replace_func_name = ['info', 'critical', 'fatal', 'debug', + 'error', 'warn', 'warning', 'findCaller'] + + +class MDCContext(threading.local): + """ + A Thread local instance to storage mdc values + """ + def __init__(self): + + super(MDCContext, self).__init__() + self._localDict = {} + + def get(self, key): + + return self._localDict.get(key, None) + + def put(self, key, value): + + self._localDict[key] = value + + def remove(self, key): + + if key in self.localDict: + del self._localDict[key] + + def clear(self): + + self._localDict.clear() + + def result(self): + + return self._localDict + + def isEmpty(self): + + return self._localDict == {} or self._localDict is None + + +MDC = MDCContext() + + +def fetchkeys(func): + + @functools.wraps(func) + def replace(*args, **kwargs): + kwargs['extra'] = _getmdcs(extra=kwargs.get('extra', None)) + func(*args, **kwargs) + return replace + + +def _getmdcs(extra=None): + """ + Put mdc dict in logging record extra filed with key 'mdc' + :param extra: dict + :return: mdc dict + """ + if MDC.isEmpty(): + return + + mdc = MDC.result() + + if extra is not None: + for key in extra: + # make sure extra key dosen't override mdckey + if key in mdc or key == 'mdc': + raise KeyError("Attempt to overwrite %r in MDC" % key) + else: + extra = {} + + extra['mdc'] = mdc + del mdc + return extra + + +@fetchkeys +def info(self, msg, *args, **kwargs): + + if self.isEnabledFor(logging.INFO): + self._log(logging.INFO, msg, args, **kwargs) + + +@fetchkeys +def debug(self, msg, *args, **kwargs): + + if self.isEnabledFor(logging.DEBUG): + self._log(logging.DEBUG, msg, args, **kwargs) + + +@fetchkeys +def warning(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.WARNING): + self._log(logging.WARNING, msg, args, **kwargs) + + +@fetchkeys +def exception(self, msg, *args, **kwargs): + + kwargs['exc_info'] = 1 + self.error(msg, *args, **kwargs) + + +@fetchkeys +def critical(self, msg, *args, **kwargs): + + if self.isEnabledFor(logging.CRITICAL): + self._log(logging.CRITICAL, msg, args, **kwargs) + + +@fetchkeys +def error(self, msg, *args, **kwargs): + if self.isEnabledFor(logging.ERROR): + self._log(logging.ERROR, msg, args, **kwargs) + + +def findCaller(self): + + f = logging.currentframe() + if f is not None: + f = f.f_back + rv = "(unkown file)", 0, "(unknow function)" + while hasattr(f, "f_code"): + co = f.f_code + filename = os.path.normcase(co.co_filename) + # jump through local 'replace' func frame + if filename == logging._srcfile or co.co_name == "replace": + f = f.f_back + continue + rv = (co.co_filename, f.f_lineno, co.co_name) + break + + return rv + + +def patch_loggingMDC(): + """ + The patch to add MDC ability in logging Record instance at runtime + """ + localModule = sys.modules[__name__] + for attr in dir(logging.Logger): + if attr in _replace_func_name: + newfunc = getattr(localModule, attr, None) + if newfunc: + setattr(logging.Logger, attr, newfunc) diff --git a/pylog/onaplogging/mdcformatter.py b/pylog/onaplogging/mdcformatter.py new file mode 100644 index 0000000..f63ec94 --- /dev/null +++ b/pylog/onaplogging/mdcformatter.py @@ -0,0 +1,123 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +import logging + + +class MDCFormatter(logging.Formatter): + """ + A custom MDC formatter to prepare Mapped Diagnostic Context + to enrich log message. + """ + + def __init__(self, fmt=None, mdcfmt=None, datefmt=None): + """ + :param fmt: build-in format string contains standard + Python %-style mapping keys + :param mdcFmt: mdc format with '{}'-style mapping keys + :param datefmt: Date format to use + """ + + super(MDCFormatter, self).__init__(fmt=fmt, datefmt=datefmt) + self._tmpfmt = self._fmt + if mdcfmt: + self._mdcFmt = mdcfmt + else: + self._mdcFmt = '{reqeustID}' + + def _mdcfmtKey(self): + """ + maximum barce match algorithm to find the mdc key + :return: key in brace and key not in brace,such as ({key}, key) + """ + + left = '{' + right = '}' + target = self._mdcFmt + st = [] + keys = [] + for index, v in enumerate(target): + if v == left: + st.append(index) + elif v == right: + + if len(st) == 0: + continue + + elif len(st) == 1: + start = st.pop() + end = index + keys.append(target[start:end + 1]) + elif len(st) > 0: + st.pop() + + keys = filter(lambda x: x[1:-1].strip('\n \t ') != "", keys) + words = None + if keys: + words = map(lambda x: x[1:-1], keys) + + return keys, words + + def _replaceStr(self, keys): + + fmt = self._mdcFmt + for i in keys: + fmt = fmt.replace(i, i[1:-1] + "=" + i) + + return fmt + + def format(self, record): + """ + Find mdcs in log record extra field, if key form mdcFmt dosen't + contains mdcs, the values will be empty. + :param record: the logging record instance + :return: string + for example: + the mdcs dict in logging record is + {'key1':'value1','key2':'value2'} + the mdcFmt is" '{key1} {key3}' + the output of mdc message: 'key1=value1 key3=' + + """ + mdcIndex = self._fmt.find('%(mdc)s') + if mdcIndex == -1: + return super(MDCFormatter, self).format(record) + + mdcFmtkeys, mdcFmtWords = self._mdcfmtKey() + if mdcFmtWords is None: + + self._fmt = self._fmt.replace("%(mdc)s", "") + return super(MDCFormatter, self).format(record) + + mdc = record.__dict__.get('mdc', None) + res = {} + for i in mdcFmtWords: + if mdc and i in mdc: + res[i] = mdc[i] + else: + res[i] = "" + + del mdc + try: + mdcstr = self._replaceStr(keys=mdcFmtkeys).format(**res) + self._fmt = self._fmt.replace("%(mdc)s", mdcstr) + s = super(MDCFormatter, self).format(record) + return s + + except KeyError as e: + print ("The mdc key %s format is wrong" % e.message) + except Exception: + raise + + finally: + # reset fmt format + self._fmt = self._tmpfmt diff --git a/pylog/onaplogging/monkey.py b/pylog/onaplogging/monkey.py new file mode 100644 index 0000000..fcf8fdf --- /dev/null +++ b/pylog/onaplogging/monkey.py @@ -0,0 +1,27 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +from mdcContext import patch_loggingMDC +from logWatchDog import patch_loggingYaml + + +__all__ = ["patch_all"] + + +def patch_all(mdc=True, yaml=True): + + if mdc is True: + patch_loggingMDC() + + if yaml is True: + patch_loggingYaml() diff --git a/pylog/pom.xml b/pylog/pom.xml new file mode 100644 index 0000000..f316069 --- /dev/null +++ b/pylog/pom.xml @@ -0,0 +1,50 @@ +<?xml version="1.0"?> +<!-- + Copyright (c) 2017-2018 VMware, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at: + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +--> +<project + xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <groupId>org.onap.logging-analytics</groupId> + <artifactId>logging-analytics</artifactId> + <version>1.2.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>logging-pylog</artifactId> + <version>1.2.0-SNAPSHOT</version> + <packaging>pom</packaging> + <name>logging-pylog</name> + <description>onap python logging library</description> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project>
\ No newline at end of file diff --git a/pylog/requirements.txt b/pylog/requirements.txt new file mode 100644 index 0000000..3fb9241 --- /dev/null +++ b/pylog/requirements.txt @@ -0,0 +1,2 @@ +PyYAML>=3.10 +watchdog>=0.8.3 diff --git a/pylog/setup.py b/pylog/setup.py new file mode 100644 index 0000000..bcd2347 --- /dev/null +++ b/pylog/setup.py @@ -0,0 +1,37 @@ +# Copyright (c) 2018 VMware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +from setuptools import setup, find_packages + +setup( + + name='onappylog', + keywords=("yaml", "logging", "mdc", "onap"), + description='onap python logging library', + long_description="python-package onappylog could be used in any python project to record MDC information and reload logging at runtime", + version="1.0.5", + license="MIT Licence", + author='ke liang', + author_email="lokyse@163.com", + packages=find_packages(), + platforms=['all'], + install_requires=[ + "PyYAML>=3.10", + "watchdog>=0.8.3" + ], + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'Programming Language :: Python :: 2.7' + ] +) diff --git a/pylog/tests/__init__.py b/pylog/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/pylog/tests/__init__.py diff --git a/pylog/tests/test_example.py b/pylog/tests/test_example.py new file mode 100644 index 0000000..c0d97bf --- /dev/null +++ b/pylog/tests/test_example.py @@ -0,0 +1,18 @@ +# Copyright (c) 2018-2019 VMware, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + + +import unittest + + +class TestExample(unittest.TestCase): + + def test_mdcFormat(self): + return diff --git a/pylog/tox.ini b/pylog/tox.ini new file mode 100644 index 0000000..a47f58c --- /dev/null +++ b/pylog/tox.ini @@ -0,0 +1,34 @@ +[tox] +envlist =py,py27,pep8 +skipsdist = true +skip_missing_interpreters = true + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv] +deps = -r{toxinidir}/requirements.txt + pytest + coverage + pytest-cov +setenv = PYTHONPATH={toxinidir}/ + +commands = + /usr/bin/find . -type f -name "*.py[c|o]" -delete + py.test + +[testenv:pep8] +deps=flake8 +commands=flake8 + +[flake8] +show-source = true +exclude = env,venv,.venv,.git,.tox,dist,doc,*egg,build + + +[testenv:py27] +commands = + {[testenv]commands} + +[testenv:cover] +commands = py.test --cov onaplogging |