3.0.0 policy-handler - migrated to python 3.6

- migrated from python 2.7 to 3.6
- brought up the latest versions of dependencies
  -- Cherrypy 15.0.0, requests 2.18.4, websocket-client 0.48.0
- fixed migration errors
  -- renamed the standard package Queue to queue
  -- dict.items() instead of dict.iteritems()
  -- dict.keys() instead of dict.viewkeys()
  -- range() instead of xrange()
  -- subprocess.check_output(..., universal_newlines=True) to
     get str instead of byte-stream from stdout
- cleaned up migration warnings
  -- super() instead of super(A, self)
  -- logger.warning() instead of .warn()
- moved main() from policy_handler.py to __main__.py
- getting the policy_handler version directly from setup.py
  instead of the env var on init of the audit

Change-Id: I0fc4ddc51c08a64f3cfdc5d2f010b1c6a1ae92f0
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-515

12 files changed, 79 insertions, 61 deletions

diff --git a/policyhandler/__init__.py b/policyhandler/__init__.py
index e9d0246..3315706 100644
--- a/policyhandler/__init__.py
+++ b/policyhandler/__init__.py
@@ -15,3 +15,20 @@
 # ============LICENSE_END=========================================================
 #
 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
+
+"""policyhandler package"""
+
+class LogWriter(object):
+    """redirect the standard out + err to the logger"""
+    def __init__(self, logger_func):
+        self.logger_func = logger_func
+
+    def write(self, log_line):
+        """actual writer to be used in place of stdout or stderr"""
+        log_line = log_line.rstrip()
+        if log_line:
+            self.logger_func(log_line)
+
+    def flush(self):
+        """no real flushing of the buffer"""
+        pass
diff --git a/policyhandler/policy_handler.py b/policyhandler/__main__.py
index 8d5f360..1f17d9d 100644
--- a/policyhandler/policy_handler.py
+++ b/policyhandler/__main__.py
@@ -16,31 +16,22 @@
 #
 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
 
-"""run as server: python -m policyhandler/policy_handler"""
+"""
+    run as server:
+    python -m policyhandler
+
+    that will invoke this module __main__.py in folder of policyhandler
+"""
 
-import os
-import sys
 import logging
+import sys
 
+from policyhandler import LogWriter
 from policyhandler.config import Config
 from policyhandler.onap.audit import Audit
-from policyhandler.web_server import PolicyWeb
 from policyhandler.policy_receiver import PolicyReceiver
+from policyhandler.web_server import PolicyWeb
 
-class LogWriter(object):
-    """redirect the standard out + err to the logger"""
-    def __init__(self, logger_func):
-        self.logger_func = logger_func
-
-    def write(self, log_line):
-        """actual writer to be used in place of stdout or stderr"""
-        log_line = log_line.rstrip()
-        if log_line:
-            self.logger_func(log_line)
-
-    def flush(self):
-        """no real flushing of the buffer"""
-        pass
 
 def run_policy_handler():
     """main run function for policy-handler"""
@@ -51,10 +42,8 @@ def run_policy_handler():
     sys.stdout = LogWriter(logger.info)
     sys.stderr = LogWriter(logger.error)
 
-    logger.info("========== run_policy_handler ==========")
-    policy_handler_version = os.getenv("APP_VER")
-    logger.info("policy_handler_version %s", policy_handler_version)
-    Audit.init(Config.get_system_name(), policy_handler_version, Config.LOGGER_CONFIG_FILE_PATH)
+    logger.info("========== run_policy_handler ========== %s", __package__)
+    Audit.init(Config.get_system_name(), Config.LOGGER_CONFIG_FILE_PATH)
 
     logger.info("starting policy_handler with config:")
     logger.info(Audit.log_json_dumps(Config.config))
@@ -63,5 +52,6 @@ def run_policy_handler():
     PolicyReceiver.run(audit)
     PolicyWeb.run_forever(audit)
 
+
 if __name__ == "__main__":
     run_policy_handler()
diff --git a/policyhandler/config.py b/policyhandler/config.py
index 8a8f418..39d528b 100644
--- a/policyhandler/config.py
+++ b/policyhandler/config.py
@@ -71,7 +71,7 @@ class Config(object):
         new_config = DiscoveryClient.get_value(discovery_key)
 
         if not new_config or not isinstance(new_config, dict):
-            Config._logger.warn("unexpected config from discovery: %s", new_config)
+            Config._logger.warning("unexpected config from discovery: %s", new_config)
             return
 
         Config._logger.debug("loaded config from discovery(%s): %s", \
@@ -92,7 +92,7 @@ class Config(object):
                 loaded_config = json.load(config_json)
 
         if not loaded_config:
-            Config._logger.warn("config not loaded from file: %s", file_path)
+            Config._logger.warning("config not loaded from file: %s", file_path)
             return
 
         Config._logger.info("config loaded from file: %s", file_path)
diff --git a/policyhandler/customize/customizer.py b/policyhandler/customize/customizer.py
index 22bf60e..38928e9 100644
--- a/policyhandler/customize/customizer.py
+++ b/policyhandler/customize/customizer.py
@@ -26,10 +26,13 @@ class Customizer(CustomizerBase):
 
     :Customizer: class is owned by the company that needs to customize the policy-handler
 
-    :override: any method defined in the CustomizerBase class to customize the behavior of the policy-handler
+    :override: any method defined in the CustomizerBase class to customize
+    the behavior of the policy-handler
 
     see README.md for the sample of the customizer.py
     """
-    def __init__(self):
-        """class that contains the customization"""
-        super(Customizer, self).__init__()
+    # def __init__(self):
+    #     """class that contains the customization"""
+    #     super().__init__()
+
+    pass
diff --git a/policyhandler/customize/customizer_base.py b/policyhandler/customize/customizer_base.py
index 97e1550..c98a9eb 100644
--- a/policyhandler/customize/customizer_base.py
+++ b/policyhandler/customize/customizer_base.py
@@ -35,7 +35,8 @@ class CustomizerBase(object):
 
     policy-hanlder is using the instance of the child Customizer class to get the overriden methods
 
-    the methods defined in this class are the placeholders and are expected to be overriden by the Customizer class
+    the methods defined in this class are the placeholders and are expected
+    to be overriden by the Customizer class
     """
 
     def __init__(self):
diff --git a/policyhandler/onap/audit.py b/policyhandler/onap/audit.py
index 0aa1c50..a007a26 100644
--- a/policyhandler/onap/audit.py
+++ b/policyhandler/onap/audit.py
@@ -140,10 +140,9 @@ class _Audit(object):
     _packages = []
 
     @staticmethod
-    def init(service_name, service_version, config_file_path):
+    def init(service_name, config_file_path):
         """init static invariants and loggers"""
         _Audit._service_name = service_name
-        _Audit._service_version = service_version
         _Audit._logger_debug = CommonLogger(config_file_path, "debug", \
             instanceUUID=_Audit._service_instance_uuid, serviceName=_Audit._service_name)
         _Audit._logger_error = CommonLogger(config_file_path, "error", \
@@ -154,7 +153,14 @@ class _Audit(object):
             instanceUUID=_Audit._service_instance_uuid, serviceName=_Audit._service_name)
         ProcessInfo.init()
         try:
-            _Audit._packages = filter(None, subprocess.check_output(["pip", "freeze"]).splitlines())
+            _Audit._service_version = subprocess.check_output(
+                ["python", "setup.py", "--version"], universal_newlines=True).strip()
+        except subprocess.CalledProcessError:
+            pass
+        try:
+            _Audit._packages = list(
+                filter(None, subprocess.check_output(["pip", "freeze"],
+                                                     universal_newlines=True).splitlines()))
         except subprocess.CalledProcessError:
             pass
 
@@ -337,10 +343,10 @@ class Audit(_Audit):
         :aud_parent: is the parent Audit - used for sub-query metrics to other systems
         :kwargs: - put any request related params into kwargs
         """
-        super(Audit, self).__init__(job_name=job_name,
-                                    request_id=request_id,
-                                    req_message=req_message,
-                                    **kwargs)
+        super().__init__(job_name=job_name,
+                         request_id=request_id,
+                         req_message=req_message,
+                         **kwargs)
 
         headers = self.kwargs.get("headers", {})
         if headers:
@@ -411,10 +417,10 @@ class Metrics(_Audit):
         :aud_parent: is the parent Audit - used for sub-query metrics to other systems
         :kwargs: - put any request related params into kwargs
         """
-        super(Metrics, self).__init__(job_name=aud_parent.job_name,
-                                      request_id=aud_parent.request_id,
-                                      req_message=aud_parent.req_message,
-                                      **aud_parent.merge_all_kwargs(**kwargs))
+        super().__init__(job_name=aud_parent.job_name,
+                         request_id=aud_parent.request_id,
+                         req_message=aud_parent.req_message,
+                         **aud_parent.merge_all_kwargs(**kwargs))
         self.aud_parent = aud_parent
         self._metrics_name = _Audit._key_format.sub(
             '_', AUDIT_METRICS + "_" + self.kwargs.get(AUDIT_TARGET_ENTITY, self.job_name))
diff --git a/policyhandler/onap/health.py b/policyhandler/onap/health.py
index e6a6f69..d117255 100644
--- a/policyhandler/onap/health.py
+++ b/policyhandler/onap/health.py
@@ -156,5 +156,5 @@ class Health(object):
     def dump(self):
         """returns dict of stats"""
         with self._lock:
-            stats = dict((k, v.dump()) for (k, v) in self._all_stats.iteritems())
+            stats = dict((k, v.dump()) for (k, v) in self._all_stats.items())
         return stats
diff --git a/policyhandler/onap/process_info.py b/policyhandler/onap/process_info.py
index 9fb6334..a22ccf9 100644
--- a/policyhandler/onap/process_info.py
+++ b/policyhandler/onap/process_info.py
@@ -38,7 +38,7 @@ def safe_operation(func):
         try:
             return func(*args, **kwargs)
         except Exception as ex:
-            return "%s: %s" % (type(ex).__name__, str(ex))
+            return {type(ex).__name__ : str(ex)}
     return wrapper
 
 
@@ -86,7 +86,7 @@ class ProcessInfo(object):
         process = psutil.Process()
         with process.oneshot():
             return dict((k, ProcessInfo.bytes_to_bibytes(v))
-                        for k, v in process.memory_full_info()._asdict().iteritems())
+                        for k, v in process.memory_full_info()._asdict().items())
 
 
     @staticmethod
@@ -94,7 +94,7 @@ class ProcessInfo(object):
     def virtual_memory():
         """calculates the virtual memory usage of the whole vm"""
         return dict((k, ProcessInfo.bytes_to_bibytes(v))
-                    for k, v in psutil.virtual_memory()._asdict().iteritems())
+                    for k, v in psutil.virtual_memory()._asdict().items())
 
 
     @staticmethod
diff --git a/policyhandler/policy_receiver.py b/policyhandler/policy_receiver.py
index dd9eea6..280e3c6 100644
--- a/policyhandler/policy_receiver.py
+++ b/policyhandler/policy_receiver.py
@@ -122,7 +122,8 @@ class _PolicyReceiver(Thread):
             message = json.loads(message)
 
             if not message or not isinstance(message, dict):
-                _PolicyReceiver._logger.warn("unexpected message from PDP: %s", json.dumps(message))
+                _PolicyReceiver._logger.warning("unexpected message from PDP: %s",
+                                                json.dumps(message))
                 return
 
             policies_updated = [(policy.get(POLICY_NAME), policy.get(POLICY_VER))
diff --git a/policyhandler/policy_rest.py b/policyhandler/policy_rest.py
index 977a9a1..16c38bb 100644
--- a/policyhandler/policy_rest.py
+++ b/policyhandler/policy_rest.py
@@ -207,7 +207,7 @@ class PolicyRest(object):
             latest_policy = None
             expect_policy_removed = (ignore_policy_names and not min_version_expected)
 
-            for retry in xrange(1, PolicyRest._policy_retry_count + 1):
+            for retry in range(1, PolicyRest._policy_retry_count + 1):
                 PolicyRest._logger.debug(str_metrics)
 
                 status_code, policy_configs = PolicyRest._pdp_get_config(
@@ -323,7 +323,7 @@ class PolicyRest(object):
             apns = [(audit, policy_id,
                      policy_to_find.get(PolicyRest.MIN_VERSION_EXPECTED),
                      policy_to_find.get(PolicyRest.IGNORE_POLICY_NAMES))
-                    for (policy_id, policy_to_find) in policies_to_find.iteritems()]
+                    for (policy_id, policy_to_find) in policies_to_find.items()]
 
             policies = None
             apns_length = len(apns)
@@ -343,13 +343,13 @@ class PolicyRest(object):
                                     if policy and policy.get(POLICY_ID))
 
             removed_policies = dict((policy_id, True)
-                                    for (policy_id, policy_to_find) in policies_to_find.iteritems()
+                                    for (policy_id, policy_to_find) in policies_to_find.items()
                                     if not policy_to_find.get(PolicyRest.MIN_VERSION_EXPECTED)
                                     and policy_to_find.get(PolicyRest.IGNORE_POLICY_NAMES)
                                     and policy_id not in updated_policies)
 
             errored_policies = dict((policy_id, policy_to_find)
-                                    for (policy_id, policy_to_find) in policies_to_find.iteritems()
+                                    for (policy_id, policy_to_find) in policies_to_find.items()
                                     if policy_id not in updated_policies
                                     and policy_id not in removed_policies)
 
@@ -413,7 +413,7 @@ class PolicyRest(object):
             audit.set_http_status_code(status_code)
             valid_policies = {}
             errored_policies = {}
-            for (policy_id, policy) in latest_policies.iteritems():
+            for (policy_id, policy) in latest_policies.items():
                 if PolicyRest._validate_policy(policy):
                     valid_policies[policy_id] = policy
                 else:
@@ -481,10 +481,10 @@ class PolicyRest(object):
 
             # latest_policies == [(valid_policies, errored_policies, errored_scope_prefix), ...]
             result[LATEST_POLICIES] = dict(
-                pair for (vps, _, _) in latest_policies if vps for pair in vps.iteritems())
+                pair for (vps, _, _) in latest_policies if vps for pair in vps.items())
 
             result[ERRORED_POLICIES] = dict(
-                pair for (_, eps, _) in latest_policies if eps for pair in eps.iteritems())
+                pair for (_, eps, _) in latest_policies if eps for pair in eps.items())
 
             result[ERRORED_SCOPES] = sorted([esp for (_, _, esp) in latest_policies if esp])
 
diff --git a/policyhandler/policy_updater.py b/policyhandler/policy_updater.py
index 9f24d3d..cff7b41 100644
--- a/policyhandler/policy_updater.py
+++ b/policyhandler/policy_updater.py
@@ -21,7 +21,7 @@
 import copy
 import json
 import logging
-from Queue import Queue
+from queue import Queue
 from threading import Lock, Thread
 
 from .config import Config
@@ -206,14 +206,14 @@ class PolicyUpdater(Thread):
             catch_up_result = ""
             if not aud_catch_up.is_success():
                 catch_up_result = "- not sending catch-up to deployment-handler due to errors"
-                PolicyUpdater._logger.warn(catch_up_result)
+                PolicyUpdater._logger.warning(catch_up_result)
             elif not self._need_to_send_catch_up(aud_catch_up, catch_up_message):
                 catch_up_result = "- skipped sending the same policies"
             else:
                 DeployHandler.policy_update(aud_catch_up, catch_up_message, rediscover=True)
                 if not aud_catch_up.is_success():
                     catch_up_result = "- failed to send catch-up to deployment-handler"
-                    PolicyUpdater._logger.warn(catch_up_result)
+                    PolicyUpdater._logger.warning(catch_up_result)
                 else:
                     catch_up_result = "- sent catch-up to deployment-handler"
             success, _, _ = aud_catch_up.audit_done(result=catch_up_result)
@@ -255,13 +255,13 @@ class PolicyUpdater(Thread):
 
             if not queued_audit.is_success():
                 result = "- not sending policy-updates to deployment-handler due to errors"
-                PolicyUpdater._logger.warn(result)
+                PolicyUpdater._logger.warning(result)
             else:
                 message = {LATEST_POLICIES: updated_policies, REMOVED_POLICIES: removed_policies}
                 deployment_handler_changed = DeployHandler.policy_update(queued_audit, message)
                 if not queued_audit.is_success():
                     result = "- failed to send policy-updates to deployment-handler"
-                    PolicyUpdater._logger.warn(result)
+                    PolicyUpdater._logger.warning(result)
                 else:
                     result = "- sent policy-updates to deployment-handler"
 
diff --git a/policyhandler/policy_utils.py b/policyhandler/policy_utils.py
index c96d4f6..c2a8b07 100644
--- a/policyhandler/policy_utils.py
+++ b/policyhandler/policy_utils.py
@@ -134,8 +134,8 @@ class Utils(object):
         try:
             return json.loads(json_str)
         except (ValueError, TypeError) as err:
-            Utils._logger.warn("unexpected json error(%s): len(%s) str[:100]: (%s)",
-                               str(err), len(json_str), str(json_str)[:100])
+            Utils._logger.warning("unexpected json error(%s): len(%s) str[:100]: (%s)",
+                                  str(err), len(json_str), str(json_str)[:100])
         return json_str
 
     @staticmethod
@@ -159,11 +159,11 @@ class Utils(object):
             return True
 
         if isinstance(body_1, dict) and isinstance(body_2, dict):
-            if body_1.viewkeys() ^ body_2.viewkeys():
+            if body_1.keys() ^ body_2.keys():
                 Utils._logger.debug("keys %s != %s", json.dumps(body_1), json.dumps(body_2))
                 return False
 
-            for key, val_1 in body_1.iteritems():
+            for key, val_1 in body_1.items():
                 if not Utils.are_the_same(val_1, body_2[key]):
                     return False
             return True