4.0.1 fix for removing the non-matching policy

- fixed removing the policy that no longer match the filter
  = sending an empty policy_filter_matches for the policy_id
- cleaner workaround on getting the updated policy
  when policy-engine sent policy-update notification
  before finishing the update on there side
  = using the collection of expected_versions instead
    of min_version_expected
- some minor refactoring on policy_matcher and audit

Change-Id: Ica3cb810378e61d6991c616f88265ff170d32a64
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-492

9 files changed, 51 insertions, 51 deletions

diff --git a/policyhandler/deploy_handler.py b/policyhandler/deploy_handler.py
index 6b7788c..3d299f3 100644
--- a/policyhandler/deploy_handler.py
+++ b/policyhandler/deploy_handler.py
@@ -73,7 +73,7 @@ class PolicyUpdateMessage(object):
         if latest_policy:
             self._latest_policies[policy_id] = deepcopy(latest_policy)
 
-        if policy_filter_ids:
+        if policy_filter_ids is not None:
             if policy_id not in self._policy_filter_matches:
                 self._policy_filter_matches[policy_id] = {}
             self._policy_filter_matches[policy_id].update(policy_filter_ids)
diff --git a/policyhandler/onap/audit.py b/policyhandler/onap/audit.py
index 69ddb86..db85c18 100644
--- a/policyhandler/onap/audit.py
+++ b/policyhandler/onap/audit.py
@@ -135,6 +135,8 @@ class _Audit(object):
     _logger_error = None
     _logger_metrics = None
     _logger_audit = None
+    _hostname = os.environ.get(HOSTNAME)
+
     _health = Health()
     _py_ver = sys.version.replace("\n", "")
     _packages = []
@@ -371,21 +373,17 @@ class Audit(_Audit):
         if headers:
             if not self.request_id:
                 self.request_id = headers.get(REQUEST_X_ECOMP_REQUESTID)
-            if AUDIT_IPADDRESS not in self.kwargs:
-                self.kwargs[AUDIT_IPADDRESS] = headers.get(REQUEST_REMOTE_ADDR)
-            if AUDIT_SERVER not in self.kwargs:
-                self.kwargs[AUDIT_SERVER] = headers.get(REQUEST_HOST)
+            self.kwargs.setdefault(AUDIT_IPADDRESS, headers.get(REQUEST_REMOTE_ADDR))
+            self.kwargs.setdefault(AUDIT_SERVER, headers.get(REQUEST_HOST))
 
         created_req = ""
         if not self.request_id:
             created_req = " with new"
             self.request_id = str(uuid.uuid4())
-
-        if AUDIT_SERVER not in self.kwargs:
-            self.kwargs[AUDIT_SERVER] = os.environ.get(HOSTNAME)
-
         self.kwargs[AUDIT_REQUESTID] = self.request_id
 
+        self.kwargs.setdefault(AUDIT_SERVER, _Audit._hostname)
+
         _Audit._health.start(self.job_name, self.request_id)
         _Audit._health.start(AUDIT_TOTAL_STATS, self.request_id)
 
diff --git a/policyhandler/policy_matcher.py b/policyhandler/policy_matcher.py
index 8406f14..71b5ce8 100644
--- a/policyhandler/policy_matcher.py
+++ b/policyhandler/policy_matcher.py
@@ -70,6 +70,13 @@ class PolicyMatcher(object):
         latest_policies = pdp_response.get(LATEST_POLICIES, {})
         errored_policies = pdp_response.get(ERRORED_POLICIES, {})
 
+        latest_policies, changed_policies, policy_filter_matches = PolicyMatcher._match_policies(
+            audit, latest_policies, deployed_policies, deployed_policy_filters)
+
+        errored_policies = dict((policy_id, policy)
+                                for (policy_id, policy) in errored_policies.items()
+                                if deployed_policies.get(policy_id, {}).get(POLICY_VERSIONS))
+
         removed_policies = dict(
             (policy_id, True)
             for (policy_id, deployed_policy) in deployed_policies.items()
@@ -78,11 +85,6 @@ class PolicyMatcher(object):
             and policy_id not in errored_policies
         )
 
-        latest_policies, changed_policies, policy_filter_matches = PolicyMatcher.match_policies(
-            audit, latest_policies, deployed_policies, deployed_policy_filters)
-        errored_policies, _, _ = PolicyMatcher.match_policies(
-            audit, errored_policies, deployed_policies, deployed_policy_filters)
-
         return ({LATEST_POLICIES: latest_policies, ERRORED_POLICIES: errored_policies},
                 PolicyUpdateMessage(changed_policies,
                                     removed_policies,
@@ -110,20 +112,21 @@ class PolicyMatcher(object):
     @staticmethod
     def match_to_deployed_policies(audit, policies_updated, policies_removed):
         """match the policies_updated, policies_removed versus deployed policies"""
-        deployed_policies, deployed_policy_filters = DeployHandler.get_deployed_policies(
-            audit)
+        deployed_policies, deployed_policy_filters = DeployHandler.get_deployed_policies(audit)
         if not audit.is_success():
             return {}, {}, {}
 
-        _, changed_policies, policy_filter_matches = PolicyMatcher.match_policies(
+        _, changed_policies, policy_filter_matches = PolicyMatcher._match_policies(
             audit, policies_updated, deployed_policies, deployed_policy_filters)
-        policies_removed, _, _ = PolicyMatcher.match_policies(
-            audit, policies_removed, deployed_policies, deployed_policy_filters)
+
+        policies_removed = dict((policy_id, policy)
+                                for (policy_id, policy) in policies_removed.items()
+                                if deployed_policies.get(policy_id, {}).get(POLICY_VERSIONS))
 
         return changed_policies, policies_removed, policy_filter_matches
 
     @staticmethod
-    def match_policies(audit, policies, deployed_policies, deployed_policy_filters):
+    def _match_policies(audit, policies, deployed_policies, deployed_policy_filters):
         """
         Match policies to deployed policies either by policy_id or the policy-filters.
 
@@ -150,10 +153,11 @@ class PolicyMatcher(object):
                                    deployed_policy.get(POLICY_VERSIONS, {}).keys()))
             if policy_changed:
                 changed_policies[policy_id] = policy
+                policy_filter_matches[policy_id] = {}
 
             in_filters = False
             for (policy_filter_id, policy_filter) in deployed_policy_filters.items():
-                if not PolicyMatcher.match_policy_to_filter(
+                if not PolicyMatcher._match_policy_to_filter(
                         audit, policy_id, policy,
                         policy_filter_id, policy_filter.get(POLICY_FILTER)):
                     continue
@@ -171,7 +175,7 @@ class PolicyMatcher(object):
         return matching_policies, changed_policies, policy_filter_matches
 
     @staticmethod
-    def match_policy_to_filter(audit, policy_id, policy, policy_filter_id, policy_filter):
+    def _match_policy_to_filter(audit, policy_id, policy, policy_filter_id, policy_filter):
         """Match the policy to the policy-filter"""
         if not policy_id or not policy or not policy_filter or not policy_filter_id:
             return False
@@ -188,7 +192,7 @@ class PolicyMatcher(object):
         if not policy_name:
             return False
 
-        log_line = "policy {} to filter id {}: {}".format(json.dumps(policy_body),
+        log_line = "policy {} to filter id {}: {}".format(json.dumps(policy),
                                                           policy_filter_id,
                                                           json.dumps(policy_filter))
         # PolicyMatcher._logger.debug(audit.debug("matching {}...".format(log_line)))
@@ -214,7 +218,7 @@ class PolicyMatcher(object):
 
         filter_config_name = policy_filter.get("configName")
         policy_config_name = matching_conditions.get("ConfigName")
-        if filter_onap_name and filter_onap_name != policy_onap_name:
+        if filter_onap_name and filter_config_name != policy_config_name:
             PolicyMatcher._logger.debug(
                 audit.debug("not match by configName: {} != {}: {}"
                             .format(policy_config_name, filter_config_name, log_line)))
diff --git a/policyhandler/policy_rest.py b/policyhandler/policy_rest.py
index 39c26a5..6ec982a 100644
--- a/policyhandler/policy_rest.py
+++ b/policyhandler/policy_rest.py
@@ -48,7 +48,7 @@ class PolicyRest(object):
     PDP_STATUS_CODE_ERROR = 400
     PDP_DATA_NOT_FOUND = "PE300 - Data Issue: Incorrect Params passed: Decision not a Permit."
 
-    MIN_VERSION_EXPECTED = "min_version_expected"
+    EXPECTED_VERSIONS = "expected_versions"
     IGNORE_POLICY_NAMES = "ignore_policy_names"
 
     _requests_session = None
@@ -203,13 +203,13 @@ class PolicyRest(object):
     @staticmethod
     def get_latest_policy(aud_policy_id):
         """safely try retrieving the latest policy for the policy_id from the policy-engine"""
-        audit, policy_id, min_version_expected, ignore_policy_names = aud_policy_id
-        str_metrics = "policy_id({0}), min_version_expected({1}) ignore_policy_names({2})".format(
-            policy_id, min_version_expected, json.dumps(ignore_policy_names))
+        audit, policy_id, expected_versions, ignore_policy_names = aud_policy_id
+        str_metrics = "policy_id({0}), expected_versions({1}) ignore_policy_names({2})".format(
+            policy_id, json.dumps(expected_versions), json.dumps(ignore_policy_names))
 
         try:
             return PolicyRest._get_latest_policy(
-                audit, policy_id, min_version_expected, ignore_policy_names, str_metrics)
+                audit, policy_id, expected_versions, ignore_policy_names, str_metrics)
 
         except Exception as ex:
             error_msg = ("{0}: crash {1} {2} at {3}: {4}"
@@ -224,19 +224,19 @@ class PolicyRest(object):
 
     @staticmethod
     def _get_latest_policy(audit, policy_id,
-                           min_version_expected, ignore_policy_names, str_metrics):
+                           expected_versions, ignore_policy_names, str_metrics):
         """retry several times getting the latest policy for the policy_id from the policy-engine"""
         PolicyRest._lazy_init()
         latest_policy = None
         status_code = 0
         retry_get_config = audit.kwargs.get("retry_get_config")
-        expect_policy_removed = (ignore_policy_names and not min_version_expected)
+        expect_policy_removed = (ignore_policy_names and not expected_versions)
 
         for retry in range(1, PolicyRest._policy_retry_count + 1):
             PolicyRest._logger.debug(str_metrics)
 
             done, latest_policy, status_code = PolicyRest._get_latest_policy_once(
-                audit, policy_id, min_version_expected, ignore_policy_names,
+                audit, policy_id, expected_versions, ignore_policy_names,
                 expect_policy_removed)
 
             if done or not retry_get_config or not PolicyRest._policy_retry_sleep:
@@ -271,7 +271,7 @@ class PolicyRest(object):
 
     @staticmethod
     def _get_latest_policy_once(audit, policy_id,
-                                min_version_expected, ignore_policy_names,
+                                expected_versions, ignore_policy_names,
                                 expect_policy_removed):
         """single attempt to get the latest policy for the policy_id from the policy-engine"""
 
@@ -281,7 +281,7 @@ class PolicyRest(object):
                                  status_code, policy_id, json.dumps(policy_bodies or []))
 
         latest_policy = PolicyUtils.select_latest_policy(
-            policy_bodies, min_version_expected, ignore_policy_names
+            policy_bodies, expected_versions, ignore_policy_names
         )
 
         if not latest_policy and not expect_policy_removed:
@@ -340,12 +340,11 @@ class PolicyRest(object):
             if not policy:
                 policies_to_find[policy_id] = {
                     POLICY_ID: policy_id,
-                    PolicyRest.MIN_VERSION_EXPECTED: int(policy_version),
+                    PolicyRest.EXPECTED_VERSIONS: {policy_version: True},
                     PolicyRest.IGNORE_POLICY_NAMES: {}
                 }
                 continue
-            if int(policy[PolicyRest.MIN_VERSION_EXPECTED]) < int(policy_version):
-                policy[PolicyRest.MIN_VERSION_EXPECTED] = int(policy_version)
+            policy[PolicyRest.EXPECTED_VERSIONS][policy_version] = True
 
         for (policy_id, policy_names) in policies_removed:
             if not policy_id:
@@ -360,7 +359,7 @@ class PolicyRest(object):
             policy[PolicyRest.IGNORE_POLICY_NAMES].update(policy_names)
 
         apns = [(audit, policy_id,
-                 policy_to_find.get(PolicyRest.MIN_VERSION_EXPECTED),
+                 policy_to_find.get(PolicyRest.EXPECTED_VERSIONS),
                  policy_to_find.get(PolicyRest.IGNORE_POLICY_NAMES))
                 for (policy_id, policy_to_find) in policies_to_find.items()]
 
@@ -383,7 +382,7 @@ class PolicyRest(object):
 
         removed_policies = dict((policy_id, True)
                                 for (policy_id, policy_to_find) in policies_to_find.items()
-                                if not policy_to_find.get(PolicyRest.MIN_VERSION_EXPECTED)
+                                if not policy_to_find.get(PolicyRest.EXPECTED_VERSIONS)
                                 and policy_to_find.get(PolicyRest.IGNORE_POLICY_NAMES)
                                 and policy_id not in updated_policies)
 
diff --git a/policyhandler/policy_updater.py b/policyhandler/policy_updater.py
index 3ae8199..7733146 100644
--- a/policyhandler/policy_updater.py
+++ b/policyhandler/policy_updater.py
@@ -302,20 +302,20 @@ class PolicyUpdater(Thread):
             else:
                 message = PolicyUpdateMessage(updated_policies, removed_policies,
                                               policy_filter_matches, False)
-                log_updates = ("policies-updated[{0}], removed[{1}]"
-                               .format(len(updated_policies), len(removed_policies)))
+                log_updates = ("policies-updated[{}], removed[{}], policy_filter_matches[{}]"
+                               .format(len(updated_policies),
+                                       len(removed_policies),
+                                       len(policy_filter_matches)))
 
                 audit.reset_http_status_not_found()
                 DeployHandler.policy_update(audit, message)
 
+                log_line = "request_id[{}]: {}".format(audit.request_id, str(message))
                 if not audit.is_success():
                     result = "- failed to send to deployment-handler {}".format(log_updates)
                     PolicyUpdater._logger.warning(result)
                 else:
                     result = "- sent to deployment-handler {}".format(log_updates)
-                    log_line = "request_id: {} updated_policies: {} removed_policies: {}".format(
-                        audit.request_id,
-                        json.dumps(updated_policies), json.dumps(removed_policies))
 
             audit.audit_done(result=result)
             PolicyUpdater._logger.info(log_line + " " + result)
diff --git a/policyhandler/policy_utils.py b/policyhandler/policy_utils.py
index c63f382..da83935 100644
--- a/policyhandler/policy_utils.py
+++ b/policyhandler/policy_utils.py
@@ -76,7 +76,7 @@ class PolicyUtils(object):
         return {POLICY_ID:policy_id, POLICY_BODY:policy_body}
 
     @staticmethod
-    def select_latest_policy(policy_bodies, min_version_expected=None, ignore_policy_names=None):
+    def select_latest_policy(policy_bodies, expected_versions=None, ignore_policy_names=None):
         """For some reason, the policy-engine returns all version of the policy_bodies.
         DCAE-Controller is only interested in the latest version
         """
@@ -88,14 +88,13 @@ class PolicyUtils(object):
             policy_version = policy_body.get(POLICY_VERSION)
             if not policy_name or not policy_version or not policy_version.isdigit():
                 continue
-            policy_version = int(policy_version)
-            if min_version_expected and policy_version < min_version_expected:
+            if expected_versions and policy_version not in expected_versions:
                 continue
             if ignore_policy_names and policy_name in ignore_policy_names:
                 continue
 
             if (not latest_policy_body
-                    or int(latest_policy_body[POLICY_VERSION]) < policy_version):
+                    or int(latest_policy_body[POLICY_VERSION]) < int(policy_version)):
                 latest_policy_body = policy_body
 
         return PolicyUtils.parse_policy_config(PolicyUtils.convert_to_policy(latest_policy_body))
diff --git a/pom.xml b/pom.xml
index 6af229e..76ae7d9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property.
   <groupId>org.onap.dcaegen2.platform</groupId>
   <artifactId>policy-handler</artifactId>
   <name>dcaegen2-platform-policy-handler</name>
-  <version>4.0.0-SNAPSHOT</version>
+  <version>4.0.1-SNAPSHOT</version>
   <url>http://maven.apache.org</url>
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/setup.py b/setup.py
index 7505fb3..3667b49 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,7 @@ from setuptools import setup
 setup(

     name='policyhandler',

     description='DCAE-Controller policy-handler to communicate with policy-engine',

-    version="4.0.0",

+    version="4.0.1",

     author='Alex Shatov',

     packages=['policyhandler'],

     zip_safe=False,

diff --git a/version.properties b/version.properties
index ddff91c..53d98c4 100644
--- a/version.properties
+++ b/version.properties
@@ -1,6 +1,6 @@
 major=4

 minor=0

-patch=0

+patch=1

 base_version=${major}.${minor}.${patch}

 release_version=${base_version}

 snapshot_version=${base_version}-SNAPSHOT