diff options
Diffstat (limited to 'osdf/utils')
| -rw-r--r-- | osdf/utils/cipherUtils.py | 59 | ||||
| -rw-r--r-- | osdf/utils/data_conversion.py | 25 | ||||
| -rw-r--r-- | osdf/utils/file_utils.py | 34 | ||||
| -rw-r--r-- | osdf/utils/interfaces.py | 26 | ||||
| -rw-r--r-- | osdf/utils/mdc_utils.py | 155 |
5 files changed, 286 insertions, 13 deletions
diff --git a/osdf/utils/cipherUtils.py b/osdf/utils/cipherUtils.py new file mode 100644 index 0000000..169f1a1 --- /dev/null +++ b/osdf/utils/cipherUtils.py @@ -0,0 +1,59 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- + +from Crypto.Cipher import AES +from osdf.config.base import osdf_config +from Crypto.Util.Padding import unpad +from Crypto.Util.Padding import pad + + +class AESCipher(object): + __instance = None + + @staticmethod + def get_instance(key = None): + if AESCipher.__instance is None: + print("Creating the singleton instance") + AESCipher(key) + return AESCipher.__instance + + def __init__(self, key=None): + if AESCipher.__instance is not None: + raise Exception("This class is a singleton!") + else: + AESCipher.__instance = self + + self.bs = 32 + if key is None: + key = osdf_config.deployment["appkey"] + + self.key = key.encode() + + def encrypt(self, data): + data = data.encode() + cipher = AES.new(self.key, AES.MODE_CBC) + ciphered_data = cipher.encrypt(pad(data, AES.block_size)) + enc = (cipher.iv.hex())+(ciphered_data.hex()) + return enc + + def decrypt(self, enc): + iv = bytes.fromhex(enc[:32]) + ciphered_data = bytes.fromhex(enc[32:]) + cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) + original_data = unpad(cipher.decrypt(ciphered_data), AES.block_size).decode() + return original_data diff --git a/osdf/utils/data_conversion.py b/osdf/utils/data_conversion.py index 2f678fa..08af3e4 100644 --- a/osdf/utils/data_conversion.py +++ b/osdf/utils/data_conversion.py @@ -16,15 +16,16 @@ # ------------------------------------------------------------------------- # -import itertools from collections import defaultdict +import itertools -from dateutil import tz from dateutil.parser import parse +from dateutil import tz def tuples_to_multi_val_dict(kvw_tuples, colnums=(0, 1)): """Given a list of k,v tuples, get a dictionary of the form k -> [v1,v2,...,vn] + :param kvw_tuples: list of k,v,w tuples (e.g. [(k1,v1,a1), (k2,v2,a2), (k1,v3,a3), (k1,v4,a4)] :param colnums: column numbers :return: a dict of str:set, something like {k1: {v1, v3, v4}, k2: {v2}} or {k1: {a1, a3, a4}, k2: {a2}} @@ -38,6 +39,7 @@ def tuples_to_multi_val_dict(kvw_tuples, colnums=(0, 1)): def tuples_to_dict(kvw_tuples, colnums=(0, 1)): """Given a list of k,v tuples, get a dictionary of the form k -> v + :param kvw_tuples: list of k,v,w tuples (e.g. [(k1,v1,a1), (k2,v2,a2), (k3,v3,a3), (k1,v4,a4)] :param colnums: column numbers :return: a dict; something like {k1: v4, k2: v2, k3: v3} (note, k1 is repeated, so last val is retained) @@ -46,13 +48,17 @@ def tuples_to_dict(kvw_tuples, colnums=(0, 1)): def utc_time_from_ts(timestamp): - """Return corresponding UTC timestamp for a given ISO timestamp (or anything that parse accepts)""" + """Return corresponding UTC timestamp for a given ISO timestamp (or anything that parse accepts) + + """ return parse(timestamp).astimezone(tz.tzutc()).strftime('%Y-%m-%d %H:%M:%S') -def list_flatten(l): - """Flatten a complex nested list of nested lists into a flat list""" - return itertools.chain(*[list_flatten(j) if isinstance(j, list) else [j] for j in l]) +def list_flatten(_l): + """Flatten a complex nested list of nested lists into a flat list + + """ + return itertools.chain(*[list_flatten(j) if isinstance(j, list) else [j] for j in _l]) text_to_symbol = { @@ -60,3 +66,10 @@ text_to_symbol = { 'less': "<", 'equal': "=" } + + +def decode_data(data): + """Decode bytes to string + + """ + return data.decode(encoding='utf-8') if isinstance(data, bytes) else data diff --git a/osdf/utils/file_utils.py b/osdf/utils/file_utils.py new file mode 100644 index 0000000..b12c17d --- /dev/null +++ b/osdf/utils/file_utils.py @@ -0,0 +1,34 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +# File related utilities + +import os +from shutil import rmtree + +from osdf.logging.osdf_logging import debug_log + + +def delete_file_folder(p): + if not p: + return + debug_log.debug('Deleting folder/file {}'.format(p)) + if os.path.isfile(p): + os.remove(p) + else: + rmtree(p, ignore_errors=True) diff --git a/osdf/utils/interfaces.py b/osdf/utils/interfaces.py index a869d6d..4fa4730 100644 --- a/osdf/utils/interfaces.py +++ b/osdf/utils/interfaces.py @@ -20,12 +20,15 @@ import json import requests import yaml -from osdf.config.base import osdf_config, creds_prefixes -from osdf.logging.osdf_logging import MH, debug_log +from osdf.config.base import creds_prefixes +from osdf.config.base import osdf_config +from osdf.logging.osdf_logging import debug_log +from osdf.logging.osdf_logging import MH def get_rest_client(request_json, service): """Get a RestClient based on request_json's callback URL and osdf_config's credentials based on service name + :param request_json: :param service: so or cm :return: rc -- RestClient @@ -53,21 +56,23 @@ class RestClient(object): """Simple REST Client that supports get/post and basic auth""" def __init__(self, userid=None, passwd=None, log_func=None, url=None, timeout=None, headers=None, - method="POST", req_id=None): + method="POST", req_id=None, verify=None): self.auth = (userid, passwd) if userid and passwd else None self.headers = headers if headers else {} self.method = method self.url = url self.log_func = log_func - self.timeout = (30, 90) if timeout is None else timeout + self.timeout = (30, 120) if timeout is None else timeout self.req_id = req_id + self.verify = verify def add_headers(self, headers): self.headers.update(headers) def request(self, url=None, method=None, asjson=True, ok_codes=(2, ), raw_response=False, noresponse=False, timeout=None, **kwargs): - """ + """Sends http request to the specified url + :param url: REST end point to query :param method: GET or POST (default is None => self.method) :param asjson: whether the expected response is in json format @@ -83,16 +88,23 @@ class RestClient(object): else: debug_log.debug("Requesting URL: {} for request ID: {}".format(url or self.url, self.req_id)) + if not url: + url = self.url + if not self.verify and url.startswith("https"): + verify = osdf_config.deployment["aaf_ca_certs"] + else: + verify = self.verify + res = requests.request(url=url or self.url, method=method or self.method, auth=self.auth, headers=self.headers, - timeout=timeout or self.timeout, **kwargs) + timeout=timeout or self.timeout, verify=verify, **kwargs) if self.log_func: self.log_func(MH.received_http_response(res)) res_code = str(res.status_code) if not any(res_code.startswith(x) for x in map(str, ok_codes)): - raise res.raise_for_status() + raise BaseException(res.raise_for_status()) if raw_response: return res diff --git a/osdf/utils/mdc_utils.py b/osdf/utils/mdc_utils.py new file mode 100644 index 0000000..c74a161 --- /dev/null +++ b/osdf/utils/mdc_utils.py @@ -0,0 +1,155 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import socket +import threading +import time +import uuid + +from flask import g +from onaplogging.mdcContext import MDC + +EMPTY = "EMPTY" + +DATE_FMT = '%Y-%m-%dT%H:%M:%S' + + +def default_server_info(): + """Populate server & server_ip_address MDC fields + + """ + # If not set or purposely set = None, then set default + if MDC.get('server') is None: + try: + server = socket.getfqdn() + except Exception: + try: + server = socket.gethostname() + except Exception: + server = '' + MDC.put('server', server) + if MDC.get('serverIPAddress') is None: + try: + server_ip_address = socket.gethostbyname(MDC.get('server')) + except Exception: + server_ip_address = "" + MDC.put('serverIPAddress', server_ip_address) + + +def default_mdc(): + """Populate default MDC fields + + """ + MDC.put('instanceUUID', generate_uuid()) + MDC.put('InvocationID', generate_uuid()) + MDC.put('serviceName', 'OOF_OSDF') + MDC.put('threadID', threading.currentThread().getName()) + default_server_info() + MDC.put('requestID', generate_uuid()) + MDC.put('partnerName', 'N/A') + MDC.put('entryTimestamp', get_time()) + + +def mdc_from_json(request_json): + """Populate MDC fields given a request in json format + + """ + if MDC.get("instanceUUID") is None: + default_mdc() + MDC.put('requestID', get_request_id(request_json)) + MDC.put('partnerName', get_partner_name(request_json)) + + +def populate_default_mdc(request): + """Populate default MDC fields given the request + + """ + if MDC.get("instanceUUID") is None: + default_mdc() + g.request_start = time.process_time() + g.empty_value = "EMPTY" + g.request_id = MDC.get("requestID") + MDC.put('serviceName', request.path) + MDC.put('IPAddress', request.headers.get('X-Forwarded-For', request.remote_addr)) + + +def populate_mdc(request): + """Populate MDC fields from the request headers + + """ + populate_default_mdc(request) + req_id = request.headers.get('X-ONAP-RequestID', g.empty_value) + request_json = request.get_json() + if req_id == g.empty_value: + req_id = get_request_id(request_json) + g.request_id = req_id + MDC.put('requestID', req_id) + MDC.put('partnerName', get_partner_name(request_json)) + + +def get_request_id(request_json): + """Get the request_id from the request + + """ + request_id = request_json['requestInfo'].get('requestId') + if not request_id: + request_id = request_json['requestInfo'].get('requestID') + return request_id + + +def generate_uuid(): + """Generate an unique uuid + + """ + return f'{uuid.uuid1()}' + + +def get_partner_name(request_json): + """Get the partnerName + + """ + partner_name = EMPTY + if 'requestInfo' in request_json: + partner_name = request_json['requestInfo'].get('sourceId', EMPTY) + return partner_name + + +def clear_mdc(): + """Clear MDC + + """ + MDC.clear() + + +def get_time(): + """Generate the invocation time string + + The dateformat is %Y-%m-%dT%H:%M:%S.sss + """ + ct = time.time() + lt = time.gmtime(ct) + msec = int((ct - int(ct)) * 1000) + return f'{time.strftime(DATE_FMT, lt)}.{msec:0>3}' + + +def set_error_details(code, desc): + """set errorCode and description + + """ + MDC.put('errorCode', code) + MDC.put('errorDescription', desc) |