diff options
Diffstat (limited to 'osdf/utils')
| -rw-r--r-- | osdf/utils/data_conversion.py | 25 | ||||
| -rw-r--r-- | osdf/utils/mdc_utils.py | 101 |
2 files changed, 113 insertions, 13 deletions
diff --git a/osdf/utils/data_conversion.py b/osdf/utils/data_conversion.py index 2f678fa..08af3e4 100644 --- a/osdf/utils/data_conversion.py +++ b/osdf/utils/data_conversion.py @@ -16,15 +16,16 @@ # ------------------------------------------------------------------------- # -import itertools from collections import defaultdict +import itertools -from dateutil import tz from dateutil.parser import parse +from dateutil import tz def tuples_to_multi_val_dict(kvw_tuples, colnums=(0, 1)): """Given a list of k,v tuples, get a dictionary of the form k -> [v1,v2,...,vn] + :param kvw_tuples: list of k,v,w tuples (e.g. [(k1,v1,a1), (k2,v2,a2), (k1,v3,a3), (k1,v4,a4)] :param colnums: column numbers :return: a dict of str:set, something like {k1: {v1, v3, v4}, k2: {v2}} or {k1: {a1, a3, a4}, k2: {a2}} @@ -38,6 +39,7 @@ def tuples_to_multi_val_dict(kvw_tuples, colnums=(0, 1)): def tuples_to_dict(kvw_tuples, colnums=(0, 1)): """Given a list of k,v tuples, get a dictionary of the form k -> v + :param kvw_tuples: list of k,v,w tuples (e.g. [(k1,v1,a1), (k2,v2,a2), (k3,v3,a3), (k1,v4,a4)] :param colnums: column numbers :return: a dict; something like {k1: v4, k2: v2, k3: v3} (note, k1 is repeated, so last val is retained) @@ -46,13 +48,17 @@ def tuples_to_dict(kvw_tuples, colnums=(0, 1)): def utc_time_from_ts(timestamp): - """Return corresponding UTC timestamp for a given ISO timestamp (or anything that parse accepts)""" + """Return corresponding UTC timestamp for a given ISO timestamp (or anything that parse accepts) + + """ return parse(timestamp).astimezone(tz.tzutc()).strftime('%Y-%m-%d %H:%M:%S') -def list_flatten(l): - """Flatten a complex nested list of nested lists into a flat list""" - return itertools.chain(*[list_flatten(j) if isinstance(j, list) else [j] for j in l]) +def list_flatten(_l): + """Flatten a complex nested list of nested lists into a flat list + + """ + return itertools.chain(*[list_flatten(j) if isinstance(j, list) else [j] for j in _l]) text_to_symbol = { @@ -60,3 +66,10 @@ text_to_symbol = { 'less': "<", 'equal': "=" } + + +def decode_data(data): + """Decode bytes to string + + """ + return data.decode(encoding='utf-8') if isinstance(data, bytes) else data diff --git a/osdf/utils/mdc_utils.py b/osdf/utils/mdc_utils.py index 14b726d..c74a161 100644 --- a/osdf/utils/mdc_utils.py +++ b/osdf/utils/mdc_utils.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------- -# Copyright (c) 2019 AT&T Intellectual Property +# Copyright (c) 2020 AT&T Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,20 +18,29 @@ import socket import threading +import time import uuid +from flask import g from onaplogging.mdcContext import MDC +EMPTY = "EMPTY" + +DATE_FMT = '%Y-%m-%dT%H:%M:%S' + def default_server_info(): + """Populate server & server_ip_address MDC fields + + """ # If not set or purposely set = None, then set default if MDC.get('server') is None: try: server = socket.getfqdn() - except Exception as err: + except Exception: try: server = socket.gethostname() - except Exception as err: + except Exception: server = '' MDC.put('server', server) if MDC.get('serverIPAddress') is None: @@ -43,26 +52,104 @@ def default_server_info(): def default_mdc(): - MDC.put('instanceUUID', uuid.uuid1()) + """Populate default MDC fields + + """ + MDC.put('instanceUUID', generate_uuid()) + MDC.put('InvocationID', generate_uuid()) MDC.put('serviceName', 'OOF_OSDF') MDC.put('threadID', threading.currentThread().getName()) default_server_info() - MDC.put('requestID', 'N/A') + MDC.put('requestID', generate_uuid()) MDC.put('partnerName', 'N/A') + MDC.put('entryTimestamp', get_time()) def mdc_from_json(request_json): - default_mdc() + """Populate MDC fields given a request in json format + + """ + if MDC.get("instanceUUID") is None: + default_mdc() MDC.put('requestID', get_request_id(request_json)) - MDC.put('partnerName', request_json['requestInfo']['sourceId']) + MDC.put('partnerName', get_partner_name(request_json)) + + +def populate_default_mdc(request): + """Populate default MDC fields given the request + + """ + if MDC.get("instanceUUID") is None: + default_mdc() + g.request_start = time.process_time() + g.empty_value = "EMPTY" + g.request_id = MDC.get("requestID") + MDC.put('serviceName', request.path) + MDC.put('IPAddress', request.headers.get('X-Forwarded-For', request.remote_addr)) + + +def populate_mdc(request): + """Populate MDC fields from the request headers + + """ + populate_default_mdc(request) + req_id = request.headers.get('X-ONAP-RequestID', g.empty_value) + request_json = request.get_json() + if req_id == g.empty_value: + req_id = get_request_id(request_json) + g.request_id = req_id + MDC.put('requestID', req_id) + MDC.put('partnerName', get_partner_name(request_json)) def get_request_id(request_json): + """Get the request_id from the request + + """ request_id = request_json['requestInfo'].get('requestId') if not request_id: request_id = request_json['requestInfo'].get('requestID') return request_id +def generate_uuid(): + """Generate an unique uuid + + """ + return f'{uuid.uuid1()}' + + +def get_partner_name(request_json): + """Get the partnerName + + """ + partner_name = EMPTY + if 'requestInfo' in request_json: + partner_name = request_json['requestInfo'].get('sourceId', EMPTY) + return partner_name + + def clear_mdc(): + """Clear MDC + + """ MDC.clear() + + +def get_time(): + """Generate the invocation time string + + The dateformat is %Y-%m-%dT%H:%M:%S.sss + """ + ct = time.time() + lt = time.gmtime(ct) + msec = int((ct - int(ct)) * 1000) + return f'{time.strftime(DATE_FMT, lt)}.{msec:0>3}' + + +def set_error_details(code, desc): + """set errorCode and description + + """ + MDC.put('errorCode', code) + MDC.put('errorDescription', desc) |