From 7167d1a0c3c30afbba61edb593580cbf7244e52c Mon Sep 17 00:00:00 2001 From: Sastry Isukapalli Date: Thu, 11 Jan 2018 14:37:39 -0500 Subject: OOF design framework seed code Issue-ID: OPTFRA-18 Change-Id: Ib44b06f7184cb49f90d5936edd82cba44068b1c8 Signed-off-by: Sastry Isukapalli --- utils/__init__.py | 0 utils/data_conversion.py | 62 ++++++++++++++++++++++++++ utils/data_types.py | 30 +++++++++++++ utils/interfaces.py | 90 ++++++++++++++++++++++++++++++++++++++ utils/local_processing.py | 43 +++++++++++++++++++ utils/programming_utils.py | 105 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 330 insertions(+) create mode 100644 utils/__init__.py create mode 100644 utils/data_conversion.py create mode 100644 utils/data_types.py create mode 100644 utils/interfaces.py create mode 100644 utils/local_processing.py create mode 100644 utils/programming_utils.py (limited to 'utils') diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/data_conversion.py b/utils/data_conversion.py new file mode 100644 index 0000000..2f678fa --- /dev/null +++ b/utils/data_conversion.py @@ -0,0 +1,62 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2015-2017 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import itertools +from collections import defaultdict + +from dateutil import tz +from dateutil.parser import parse + + +def tuples_to_multi_val_dict(kvw_tuples, colnums=(0, 1)): + """Given a list of k,v tuples, get a dictionary of the form k -> [v1,v2,...,vn] + :param kvw_tuples: list of k,v,w tuples (e.g. [(k1,v1,a1), (k2,v2,a2), (k1,v3,a3), (k1,v4,a4)] + :param colnums: column numbers + :return: a dict of str:set, something like {k1: {v1, v3, v4}, k2: {v2}} or {k1: {a1, a3, a4}, k2: {a2}} + """ + res = defaultdict(set) + for x in kvw_tuples: + key, val = x[colnums[0]], x[colnums[1]] + res[key].add(val) + return dict((k, set(v)) for k, v in res.items()) + + +def tuples_to_dict(kvw_tuples, colnums=(0, 1)): + """Given a list of k,v tuples, get a dictionary of the form k -> v + :param kvw_tuples: list of k,v,w tuples (e.g. [(k1,v1,a1), (k2,v2,a2), (k3,v3,a3), (k1,v4,a4)] + :param colnums: column numbers + :return: a dict; something like {k1: v4, k2: v2, k3: v3} (note, k1 is repeated, so last val is retained) + """ + return dict((x[colnums[0]], x[colnums[1]]) for x in kvw_tuples) + + +def utc_time_from_ts(timestamp): + """Return corresponding UTC timestamp for a given ISO timestamp (or anything that parse accepts)""" + return parse(timestamp).astimezone(tz.tzutc()).strftime('%Y-%m-%d %H:%M:%S') + + +def list_flatten(l): + """Flatten a complex nested list of nested lists into a flat list""" + return itertools.chain(*[list_flatten(j) if isinstance(j, list) else [j] for j in l]) + + +text_to_symbol = { + 'greater': ">", + 'less': "<", + 'equal': "=" +} diff --git a/utils/data_types.py b/utils/data_types.py new file mode 100644 index 0000000..877d4a1 --- /dev/null +++ b/utils/data_types.py @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2015-2017 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import collections + + +def list_like(obj): + """Check if an object is a list-like object, but not a string or dict""" + return isinstance(obj, collections.Sequence) and not isinstance(obj, (str, bytes)) + + +def dict_like(obj): + """Check if an object is a list-like object, but not a string or dict""" + return isinstance(obj, collections.Mapping) + diff --git a/utils/interfaces.py b/utils/interfaces.py new file mode 100644 index 0000000..7a0e3a9 --- /dev/null +++ b/utils/interfaces.py @@ -0,0 +1,90 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2015-2017 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import requests + +from osdf.config.base import osdf_config, creds_prefixes +from osdf.logging.osdf_logging import MH, debug_log + + +def get_rest_client(request_json, service): + """Get a RestClient based on request_json's callback URL and osdf_config's credentials based on service name + :param request_json: + :param service: so or cm + :return: rc -- RestClient + """ + callback_url = request_json["requestInfo"]["callbackUrl"] + prefix = creds_prefixes[service] + config = osdf_config.deployment + c_userid, c_passwd = config[prefix + "Username"], config[prefix + "Password"] + return RestClient(url=callback_url, userid=c_userid, passwd=c_passwd) + + +class RestClient(object): + """Simple REST Client that supports get/post and basic auth""" + + def __init__(self, userid=None, passwd=None, log_func=None, url=None, timeout=None, headers=None, + method="POST", req_id=None): + self.auth = (userid, passwd) if userid and passwd else None + self.headers = headers if headers else {} + self.method = method + self.url = url + self.log_func = log_func + self.timeout = (30, 90) if timeout is None else timeout + self.req_id = req_id + + def add_headers(self, headers): + self.headers.update(headers) + + def request(self, url=None, method=None, asjson=True, ok_codes=(2, ), + raw_response=False, noresponse=False, timeout=None, **kwargs): + """ + :param url: REST end point to query + :param method: GET or POST (default is None => self.method) + :param asjson: whether the expected response is in json format + :param ok_codes: expected codes (prefix matching -- e.g. can be (20, 21, 32) or (2, 3)) + :param noresponse: If no response is expected (as long as response codes are OK) + :param raw_response: If we need just the raw response (e.g. conductor sends transaction IDs in headers) + :param timeout: Connection and read timeouts + :param kwargs: Other parameters + :return: + """ + if not self.req_id: + debug_log.debug("Requesting URL: {}".format(url or self.url)) + else: + debug_log.debug("Requesting URL: {} for request ID: {}".format(url or self.url, self.req_id)) + + res = requests.request(url=url or self.url, method=method or self.method, + auth=self.auth, headers=self.headers, + timeout=timeout or self.timeout, **kwargs) + + if self.log_func: + self.log_func(MH.received_http_response(res)) + + res_code = str(res.status_code) + if not any(res_code.startswith(x) for x in map(str, ok_codes)): + raise res.raise_for_status() + + if raw_response: + return res + elif noresponse: + return None + elif asjson: + return res.json() + else: + return res.content diff --git a/utils/local_processing.py b/utils/local_processing.py new file mode 100644 index 0000000..6768839 --- /dev/null +++ b/utils/local_processing.py @@ -0,0 +1,43 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2015-2017 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import os + +from osdf.logging.osdf_logging import metrics_log, MH, warn_audit_error + + +def local_create_job_file(req_id, json_req, fname='osdf-req-data.json'): + """Creates a "work" folder for local processing and place relevant + job task file in there""" + + work_dir = 'osdf-optim/work/' + req_id + work_file = '{}/{}'.format(work_dir, fname) + try: + cur_task = "Making a local directory in the OSDF manager for req-id: {}".format(req_id) + metrics_log.info(MH.creating_local_env(cur_task)) + os.makedirs(work_dir, exist_ok=True) + except Exception as err: + warn_audit_error(MH.error_local_env(req_id, "Can't create directory {}".format(work_dir), err)) + return None + try: + with open(work_file, 'w') as fid: + fid.write(json_req['payload']) + return work_dir + except Exception as err: + warn_audit_error(MH.error_local_env(req_id, "can't create file {}".format(work_file), err)) + return None diff --git a/utils/programming_utils.py b/utils/programming_utils.py new file mode 100644 index 0000000..a0a8fde --- /dev/null +++ b/utils/programming_utils.py @@ -0,0 +1,105 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2015-2017 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import collections +import itertools + + +class DotDict(dict): + """A dot-dict mixin to be able to access a dictionary via dot notation + source: https://stackoverflow.com/questions/2352181/how-to-use-a-dot-to-access-members-of-dictionary + """ + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + + +class MetaSingleton(type): + """Singleton class (2nd Chapter) from Learning Python Design Patterns - 2nd ed. + Chetan Giridhar, Packt Publ. 2016""" + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(MetaSingleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + + +def namedtuple_with_defaults(typename, field_names, default_values=()): + """A namedtuple with default values -- Stack overflow recipe from Mark Lodato + http://stackoverflow.com/questions/11351032/named-tuple-and-optional-keyword-arguments + :param typename: Name for the class (same as for namedtuple) + :param field_names: Field names (same as for namedtuple) + :param default_values: Can be specified as a dictionary or as a list + :return: New namedtuple object + """ + T = collections.namedtuple(typename, field_names) + T.__new__.__defaults__ = (None,) * len(T._fields) + if isinstance(default_values, collections.Mapping): + prototype = T(**default_values) + else: + prototype = T(*default_values) + T.__new__.__defaults__ = tuple(prototype) + return T + + +def dot_notation(dict_like, dot_spec): + """Return the value corresponding to the dot_spec from a dict_like object + :param dict_like: dictionary, JSON, etc. + :param dot_spec: a dot notation (e.g. a1.b1.c1.d1 => a1["b1"]["c1"]["d1"]) + :return: the value referenced by the dot_spec + """ + attrs = dot_spec.split(".") # we split the path + parent = dict_like.get(attrs[0]) + children = ".".join(attrs[1:]) + if not (parent and children): # if no children or no parent, bail out + return parent + if isinstance(parent, list): # here, we apply remaining path spec to all children + return [dot_notation(j, children) for j in parent] + elif isinstance(parent, dict): + return dot_notation(parent, children) + else: + return None + + +def list_flatten(l): + """ + Flatten a complex nested list of nested lists into a flat list (DFS). + For example, [ [1, 2], [[[2,3,4], [2,3,4]], [3,4,5, 'hello']]] + will produce [1, 2, 2, 3, 4, 2, 3, 4, 3, 4, 5, 'hello'] + """ + return list(itertools.chain(*[list_flatten(j) if isinstance(j, list) else [j] for j in l])) + + +def inverted_dict(keys: list, key_val_dict: dict) -> dict: + """ + Get val -> [keys] mapping for the given keys using key_val_dict + :param keys: the keys we are interested in (a list) + :param key_val_dict: the key -> val mapping + :return: inverted dictionary of val -> [keys] (for the subset dict of given keys) + """ + res = {} + all_tuples = ((k, key_val_dict[k] if k in key_val_dict else 'no-parent-' + k) for k in keys) + for k, v in all_tuples: + if v in res: + res[v].append(k) + else: + res[v] = [k] + # making sure to remove duplicate keys + res = dict((v, list(set(k_list))) for v, k_list in res.items()) + return res -- cgit 1.2.3-korg