diff options
Diffstat (limited to 'osdf/logging')
| -rw-r--r-- | osdf/logging/__init__.py | 4 | ||||
| -rw-r--r-- | osdf/logging/monkey.py | 35 | ||||
| -rw-r--r-- | osdf/logging/oof_mdc_context.py | 170 | ||||
| -rw-r--r-- | osdf/logging/oof_mdc_formatter.py | 51 | ||||
| -rwxr-xr-x | osdf/logging/osdf_logging.py | 112 |
5 files changed, 360 insertions, 12 deletions
diff --git a/osdf/logging/__init__.py b/osdf/logging/__init__.py index 4b25e5b..df7613d 100644 --- a/osdf/logging/__init__.py +++ b/osdf/logging/__init__.py @@ -15,3 +15,7 @@ # # ------------------------------------------------------------------------- # + +import yaml + +yaml.warnings({'YAMLLoadWarning': False}) diff --git a/osdf/logging/monkey.py b/osdf/logging/monkey.py new file mode 100644 index 0000000..f6041bc --- /dev/null +++ b/osdf/logging/monkey.py @@ -0,0 +1,35 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + + +__all__ = ["patch_all"] + +from onaplogging.logWatchDog import patch_loggingYaml + +from osdf.logging.oof_mdc_context import patch_logging_mdc + + +def patch_all(mdc=True, yaml=True): + """monkey patch osdf logging to enable mdc + + """ + if mdc is True: + patch_logging_mdc() + + if yaml is True: + patch_loggingYaml() diff --git a/osdf/logging/oof_mdc_context.py b/osdf/logging/oof_mdc_context.py new file mode 100644 index 0000000..9c9b52c --- /dev/null +++ b/osdf/logging/oof_mdc_context.py @@ -0,0 +1,170 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import logging +import re +import sys + +from onaplogging.marker import Marker +from onaplogging.marker import MARKER_TAG +from onaplogging.mdcContext import _replace_func_name +from onaplogging.mdcContext import fetchkeys +from onaplogging.mdcContext import findCaller as fc +from onaplogging.mdcContext import MDC + +from osdf.utils.mdc_utils import set_error_details + + +def findCaller(self, stack_info=False, stacklevel=1): + """replacing onaplogging.mdcContext with this method to work with py3.8 + + """ + return fc(stack_info) + + +def mdc_mapper(): + """Convert the MDC dict into comma separated, name=value string + + :return: string format + """ + return ','.join(f'{k}={v}' for (k, v) in MDC.result().items() if k not in ['customField2']) + + +@fetchkeys +def info(self, msg, *args, **kwargs): + """Wrapper method for log.info is called + + """ + if self.isEnabledFor(logging.INFO): + MDC.put('customField2', mdc_mapper()) + self._log(logging.INFO, no_sep(msg), args, **kwargs) + + +@fetchkeys +def debug(self, msg, *args, **kwargs): + """Wrapper method for log.debug is called + + msg: log message + args: logging args + kwargs: all the optional args + """ + if self.isEnabledFor(logging.DEBUG): + self._log(logging.DEBUG, no_sep(msg), args, **kwargs) + + +@fetchkeys +def warning(self, msg, *args, **kwargs): + """Wrapper method for log.warning is called + + msg: log message + args: logging args + kwargs: all the optional args + """ + if self.isEnabledFor(logging.WARNING): + self._log(logging.WARNING, no_sep(msg), args, **kwargs) + + +@fetchkeys +def exception(self, msg, *args, **kwargs): + """Wrapper method for log.exception is called + + msg: log message + args: logging args + kwargs: all the optional args + """ + kwargs['exc_info'] = 1 + self.error(no_sep(msg), *args, **kwargs) + + +@fetchkeys +def critical(self, msg, *args, **kwargs): + """Wrapper method for log.critical + + msg: log message + args: logging args + kwargs: all the optional args + """ + if self.isEnabledFor(logging.CRITICAL): + self._log(logging.CRITICAL, no_sep(msg), args, **kwargs) + + +@fetchkeys +def error(self, msg, *args, **kwargs): + """Wrapper method for log.error is called + + msg: log message + args: logging args + kwargs: all the optional args + """ + if self.isEnabledFor(logging.ERROR): + if not MDC.get('errorCode'): + set_error_details(400, 'Internal Error') + MDC.put('customField2', mdc_mapper()) + self._log(logging.ERROR, no_sep(msg), args, **kwargs) + + +@fetchkeys +def log(self, level, msg, *args, **kwargs): + """Wrapper method for log.log is called + + msg: log message + args: logging args + kwargs: all the optional args + """ + if not isinstance(level, int): + if logging.raiseExceptions: + raise TypeError("level must be an integer") + else: + return + + if self.isEnabledFor(level): + self._log(level, no_sep(msg), args, **kwargs) + + +def handle(self, record): + """Wrapper method for log.handle is called + + """ + c_marker = getattr(self, MARKER_TAG, None) + + if isinstance(c_marker, Marker): + setattr(record, MARKER_TAG, c_marker) + + if (not self.disabled) and self.filter(record): + self.callHandlers(record) + + +def no_sep(message): + """This method will remove newline, | from the message + + """ + if message is None: + return '' + return re.sub(r'[\|\n]', ' ', str(message)) + + +def patch_logging_mdc(): + """The patch to add MDC ability in logging Record instance at runtime + + """ + local_module = sys.modules[__name__] + for attr in dir(logging.Logger): + if attr in _replace_func_name: + new_func = getattr(local_module, attr, None) + if new_func: + setattr(logging.Logger, attr, new_func) diff --git a/osdf/logging/oof_mdc_formatter.py b/osdf/logging/oof_mdc_formatter.py new file mode 100644 index 0000000..6272a18 --- /dev/null +++ b/osdf/logging/oof_mdc_formatter.py @@ -0,0 +1,51 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import time + +from onaplogging.colorFormatter import RESET +from onaplogging.mdcformatter import MDCFormatter + + +class OOFMDCFormatter(MDCFormatter): + """ONAP MDC formatter + + """ + + def __init__(self, fmt=None, mdcfmt=None, + datefmt=None, colorfmt=None, style="%"): + super().__init__(fmt, mdcfmt, datefmt, colorfmt, style) + self.converter = time.gmtime + + def _replaceStr(self, keys): + """overriding the default behavior + + keys: keys that needs to be substituted + + """ + fmt = self._mdcFmt + for i in keys: + fmt = fmt.replace(i, i) + + return fmt + + def format(self, record): + """Removing the color format end of line character. + + """ + return super(OOFMDCFormatter, self).format(record).replace(RESET, '') diff --git a/osdf/logging/osdf_logging.py b/osdf/logging/osdf_logging.py index a5a9739..7ebaa99 100755 --- a/osdf/logging/osdf_logging.py +++ b/osdf/logging/osdf_logging.py @@ -15,15 +15,14 @@ # # ------------------------------------------------------------------------- # - import logging +from logging import config import os import traceback -from logging import config import yaml -from onaplogging import monkey +from osdf.logging import monkey from osdf.utils.programming_utils import MetaSingleton BASE_DIR = os.path.dirname(__file__) @@ -32,6 +31,7 @@ LOGGING_FILE = os.path.join(BASE_DIR, '..', '..', 'config', 'log.yml') def format_exception(err, prefix=None): """Format operation for use with ecomp logging + :param err: exception object :param prefix: prefix string message :return: formatted exception (via repr(traceback.format_tb(err.__traceback__)) @@ -50,9 +50,11 @@ def create_log_dirs(): os.makedirs(os.path.dirname(a['filename']), exist_ok=True) -class OOF_OSDFLogMessageHelper(metaclass=MetaSingleton): +class OOFOSDFLogMessageHelper(metaclass=MetaSingleton): """Provides loggers as a singleton (otherwise, we end up with duplicate messages). + Provides error_log, metric_log, audit_log, and debug_log (in that order) + Additionally can provide specific log handlers too """ log_handlers = None @@ -60,13 +62,13 @@ class OOF_OSDFLogMessageHelper(metaclass=MetaSingleton): def get_handlers(self, levels=None): """Return ONAP-compliant log handlers for different levels. Each "level" ends up in a different log file + with a prefix of that level. For example: error_log, metrics_log, audit_log, debug_log in that order + :param levels: None or list of levels subset of self.default_levels (["error", "metrics", "audit", "debug"]) - :param log_version: Currently only pre_onap is supported - :param config_file: Logging configuration file for ONAP compliant logging - :param service_name: Name of the service + :return: list of log_handlers in the order of levels requested. if levels is None: we return handlers for self.default_levels if levels is ["error", "audit"], we return log handlers for that. @@ -78,154 +80,240 @@ class OOF_OSDFLogMessageHelper(metaclass=MetaSingleton): return [logging.getLogger(x) for x in wanted_levels] -class OOF_OSDFLogMessageFormatter(object): +class OOFOSDFLogMessageFormatter(object): @staticmethod def accepted_valid_request(req_id, request): + """valid request message formatter + + """ return "Accepted valid request for ID: {} for endpoint: {}".format( req_id, request.url) @staticmethod def invalid_request(req_id, err): + """invalid request message formatter + + """ return "Invalid request for request ID: {}; cause: {}".format( req_id, format_exception(err)) @staticmethod def invalid_response(req_id, err): + """invalid response message formatter + + """ return "Invalid response for request ID: {}; cause: {}".format( req_id, format_exception(err)) @staticmethod def malformed_request(request, err): + """Malformed request message formatter + + """ return "Malformed request for URL {}, from {}; cause: {}".format( request.url, request.remote_address, format_exception(err)) @staticmethod def malformed_response(response, client, err): + """Malformed response message formatter + + """ return "Malformed response {} for client {}; cause: {}".format( response, client, format_exception(err)) @staticmethod def need_policies(req_id): + """Policies needed message formatter + + """ return "Policies required but found no policies for request ID: {}".format(req_id) @staticmethod def policy_service_error(url, req_id, err): + """policy service error message formatter + + """ return "Unable to call policy for {} for request ID: {}; cause: {}".format( url, req_id, format_exception(err)) @staticmethod def requesting_url(url, req_id): + """Message formatter for requesting url + + """ return "Making a call to URL {} for request ID: {}".format(url, req_id) @staticmethod def requesting(service_name, req_id): + """Message formatter for requesting a service + + """ return "Making a call to service {} for request ID: {}".format(service_name, req_id) @staticmethod def error_requesting(service_name, req_id, err): + """Message formatter on error requesting a service + + """ return "Error while requesting service {} for request ID: {}; cause: {}".format( service_name, req_id, format_exception(err)) @staticmethod def calling_back(req_id, callback_url): + """Message formatter when a callback url is invoked + + """ return "Posting result to callback URL for request ID: {}; callback URL={}".format( req_id, callback_url) @staticmethod def calling_back_with_body(req_id, callback_url, body): + """Message formatter when a callback url with body is invoked + + """ return "Posting result to callback URL for request ID: {}; callback URL={} body={}".format( req_id, callback_url, body) @staticmethod def error_calling_back(req_id, callback_url, err): + """Message formatter on an error while posting result to callback url + + """ return "Error while posting result to callback URL {} for request ID: {}; cause: {}".format( req_id, callback_url, format_exception(err)) @staticmethod def received_request(url, remote_addr, json_body): + """Message when a call is received + + """ return "Received a call to {} from {} {}".format(url, remote_addr, json_body) @staticmethod def new_worker_thread(req_id, extra_msg=""): + """Message on invoking a new worker thread + + """ res = "Initiating new worker thread for request ID: {}".format(req_id) return res + extra_msg @staticmethod def inside_worker_thread(req_id, extra_msg=""): + """Message when inside a worker thread + + """ res = "Inside worker thread for request ID: {}".format(req_id) return res + extra_msg @staticmethod def processing(req_id, desc): + """Processing a request + + """ return "Processing request ID: {} -- {}".format(req_id, desc) @staticmethod def processed(req_id, desc): + """Processed the request + + """ return "Processed request ID: {} -- {}".format(req_id, desc) @staticmethod def error_while_processing(req_id, desc, err): + """Error while processing the request + + """ return "Error while processing request ID: {} -- {}; cause: {}".format( req_id, desc, format_exception(err)) @staticmethod def creating_local_env(req_id): + """Creating a local environment + + """ return "Creating local environment request ID: {}".format( req_id) @staticmethod def error_local_env(req_id, desc, err): + """Error creating a local env + + """ return "Error while creating local environment for request ID: {} -- {}; cause: {}".format( req_id, desc, err.__traceback__) @staticmethod def inside_new_thread(req_id, extra_msg=""): + """Inside a new thread + + """ res = "Spinning up a new thread for request ID: {}".format(req_id) return res + " " + extra_msg @staticmethod def error_response_posting(req_id, desc, err): + """Error while posting a response + + """ return "Error while posting a response for a request ID: {} -- {}; cause: {}".format( req_id, desc, err.__traceback__) @staticmethod def received_http_response(resp): - """ """ + """Received a http response + + """ return "Received response [code: {}, headers: {}, data: {}]".format( resp.status_code, resp.headers, resp.__dict__) @staticmethod def sending_response(req_id, desc): + """sending a response + + """ return "Response is sent for request ID: {} -- {}".format( req_id, desc) @staticmethod def listening_response(req_id, desc): + """Resposne is sent for a request ID + + """ return "Response is sent for request ID: {} -- {}".format( req_id, desc) @staticmethod def items_received(item_num, item_type, desc="Received"): + """Items received + + """ return "{} {} {}".format(desc, item_num, item_type) @staticmethod def items_sent(item_num, item_type, desc="Published"): + """items published + + """ return "{} {} {}".format(desc, item_num, item_type) -MH = OOF_OSDFLogMessageFormatter +MH = OOFOSDFLogMessageFormatter -error_log, metrics_log, audit_log, debug_log = OOF_OSDFLogMessageHelper().get_handlers() +error_log, metrics_log, audit_log, debug_log = OOFOSDFLogMessageHelper().get_handlers() def warn_audit_error(msg): - """Log the message to error_log.warn and audit_log.warn""" + """Log the message to error_log.warn and audit_log.warn + + """ log_message_multi(msg, audit_log.warn, error_log.warn) def log_message_multi(msg, *logger_methods): """Log the msg to multiple loggers + :param msg: message to log :param logger_methods: e.g. error_log.warn, audit_log.warn, etc. """ |