From 1c4c6626f2abb24bcb35af581c6e57130f0ccae2 Mon Sep 17 00:00:00 2001 From: Alex Shatov Date: Fri, 10 Aug 2018 11:03:02 -0400 Subject: 4.0.1 fix for removing the non-matching policy - fixed removing the policy that no longer match the filter = sending an empty policy_filter_matches for the policy_id - cleaner workaround on getting the updated policy when policy-engine sent policy-update notification before finishing the update on there side = using the collection of expected_versions instead of min_version_expected - some minor refactoring on policy_matcher and audit Change-Id: Ica3cb810378e61d6991c616f88265ff170d32a64 Signed-off-by: Alex Shatov Issue-ID: DCAEGEN2-492 --- policyhandler/deploy_handler.py | 2 +- policyhandler/onap/audit.py | 14 ++++++-------- policyhandler/policy_matcher.py | 34 +++++++++++++++++++--------------- policyhandler/policy_rest.py | 29 ++++++++++++++--------------- policyhandler/policy_updater.py | 10 +++++----- policyhandler/policy_utils.py | 7 +++---- pom.xml | 2 +- setup.py | 2 +- version.properties | 2 +- 9 files changed, 51 insertions(+), 51 deletions(-) diff --git a/policyhandler/deploy_handler.py b/policyhandler/deploy_handler.py index 6b7788c..3d299f3 100644 --- a/policyhandler/deploy_handler.py +++ b/policyhandler/deploy_handler.py @@ -73,7 +73,7 @@ class PolicyUpdateMessage(object): if latest_policy: self._latest_policies[policy_id] = deepcopy(latest_policy) - if policy_filter_ids: + if policy_filter_ids is not None: if policy_id not in self._policy_filter_matches: self._policy_filter_matches[policy_id] = {} self._policy_filter_matches[policy_id].update(policy_filter_ids) diff --git a/policyhandler/onap/audit.py b/policyhandler/onap/audit.py index 69ddb86..db85c18 100644 --- a/policyhandler/onap/audit.py +++ b/policyhandler/onap/audit.py @@ -135,6 +135,8 @@ class _Audit(object): _logger_error = None _logger_metrics = None _logger_audit = None + _hostname = os.environ.get(HOSTNAME) + _health = Health() _py_ver = sys.version.replace("\n", "") _packages = [] @@ -371,21 +373,17 @@ class Audit(_Audit): if headers: if not self.request_id: self.request_id = headers.get(REQUEST_X_ECOMP_REQUESTID) - if AUDIT_IPADDRESS not in self.kwargs: - self.kwargs[AUDIT_IPADDRESS] = headers.get(REQUEST_REMOTE_ADDR) - if AUDIT_SERVER not in self.kwargs: - self.kwargs[AUDIT_SERVER] = headers.get(REQUEST_HOST) + self.kwargs.setdefault(AUDIT_IPADDRESS, headers.get(REQUEST_REMOTE_ADDR)) + self.kwargs.setdefault(AUDIT_SERVER, headers.get(REQUEST_HOST)) created_req = "" if not self.request_id: created_req = " with new" self.request_id = str(uuid.uuid4()) - - if AUDIT_SERVER not in self.kwargs: - self.kwargs[AUDIT_SERVER] = os.environ.get(HOSTNAME) - self.kwargs[AUDIT_REQUESTID] = self.request_id + self.kwargs.setdefault(AUDIT_SERVER, _Audit._hostname) + _Audit._health.start(self.job_name, self.request_id) _Audit._health.start(AUDIT_TOTAL_STATS, self.request_id) diff --git a/policyhandler/policy_matcher.py b/policyhandler/policy_matcher.py index 8406f14..71b5ce8 100644 --- a/policyhandler/policy_matcher.py +++ b/policyhandler/policy_matcher.py @@ -70,6 +70,13 @@ class PolicyMatcher(object): latest_policies = pdp_response.get(LATEST_POLICIES, {}) errored_policies = pdp_response.get(ERRORED_POLICIES, {}) + latest_policies, changed_policies, policy_filter_matches = PolicyMatcher._match_policies( + audit, latest_policies, deployed_policies, deployed_policy_filters) + + errored_policies = dict((policy_id, policy) + for (policy_id, policy) in errored_policies.items() + if deployed_policies.get(policy_id, {}).get(POLICY_VERSIONS)) + removed_policies = dict( (policy_id, True) for (policy_id, deployed_policy) in deployed_policies.items() @@ -78,11 +85,6 @@ class PolicyMatcher(object): and policy_id not in errored_policies ) - latest_policies, changed_policies, policy_filter_matches = PolicyMatcher.match_policies( - audit, latest_policies, deployed_policies, deployed_policy_filters) - errored_policies, _, _ = PolicyMatcher.match_policies( - audit, errored_policies, deployed_policies, deployed_policy_filters) - return ({LATEST_POLICIES: latest_policies, ERRORED_POLICIES: errored_policies}, PolicyUpdateMessage(changed_policies, removed_policies, @@ -110,20 +112,21 @@ class PolicyMatcher(object): @staticmethod def match_to_deployed_policies(audit, policies_updated, policies_removed): """match the policies_updated, policies_removed versus deployed policies""" - deployed_policies, deployed_policy_filters = DeployHandler.get_deployed_policies( - audit) + deployed_policies, deployed_policy_filters = DeployHandler.get_deployed_policies(audit) if not audit.is_success(): return {}, {}, {} - _, changed_policies, policy_filter_matches = PolicyMatcher.match_policies( + _, changed_policies, policy_filter_matches = PolicyMatcher._match_policies( audit, policies_updated, deployed_policies, deployed_policy_filters) - policies_removed, _, _ = PolicyMatcher.match_policies( - audit, policies_removed, deployed_policies, deployed_policy_filters) + + policies_removed = dict((policy_id, policy) + for (policy_id, policy) in policies_removed.items() + if deployed_policies.get(policy_id, {}).get(POLICY_VERSIONS)) return changed_policies, policies_removed, policy_filter_matches @staticmethod - def match_policies(audit, policies, deployed_policies, deployed_policy_filters): + def _match_policies(audit, policies, deployed_policies, deployed_policy_filters): """ Match policies to deployed policies either by policy_id or the policy-filters. @@ -150,10 +153,11 @@ class PolicyMatcher(object): deployed_policy.get(POLICY_VERSIONS, {}).keys())) if policy_changed: changed_policies[policy_id] = policy + policy_filter_matches[policy_id] = {} in_filters = False for (policy_filter_id, policy_filter) in deployed_policy_filters.items(): - if not PolicyMatcher.match_policy_to_filter( + if not PolicyMatcher._match_policy_to_filter( audit, policy_id, policy, policy_filter_id, policy_filter.get(POLICY_FILTER)): continue @@ -171,7 +175,7 @@ class PolicyMatcher(object): return matching_policies, changed_policies, policy_filter_matches @staticmethod - def match_policy_to_filter(audit, policy_id, policy, policy_filter_id, policy_filter): + def _match_policy_to_filter(audit, policy_id, policy, policy_filter_id, policy_filter): """Match the policy to the policy-filter""" if not policy_id or not policy or not policy_filter or not policy_filter_id: return False @@ -188,7 +192,7 @@ class PolicyMatcher(object): if not policy_name: return False - log_line = "policy {} to filter id {}: {}".format(json.dumps(policy_body), + log_line = "policy {} to filter id {}: {}".format(json.dumps(policy), policy_filter_id, json.dumps(policy_filter)) # PolicyMatcher._logger.debug(audit.debug("matching {}...".format(log_line))) @@ -214,7 +218,7 @@ class PolicyMatcher(object): filter_config_name = policy_filter.get("configName") policy_config_name = matching_conditions.get("ConfigName") - if filter_onap_name and filter_onap_name != policy_onap_name: + if filter_onap_name and filter_config_name != policy_config_name: PolicyMatcher._logger.debug( audit.debug("not match by configName: {} != {}: {}" .format(policy_config_name, filter_config_name, log_line))) diff --git a/policyhandler/policy_rest.py b/policyhandler/policy_rest.py index 39c26a5..6ec982a 100644 --- a/policyhandler/policy_rest.py +++ b/policyhandler/policy_rest.py @@ -48,7 +48,7 @@ class PolicyRest(object): PDP_STATUS_CODE_ERROR = 400 PDP_DATA_NOT_FOUND = "PE300 - Data Issue: Incorrect Params passed: Decision not a Permit." - MIN_VERSION_EXPECTED = "min_version_expected" + EXPECTED_VERSIONS = "expected_versions" IGNORE_POLICY_NAMES = "ignore_policy_names" _requests_session = None @@ -203,13 +203,13 @@ class PolicyRest(object): @staticmethod def get_latest_policy(aud_policy_id): """safely try retrieving the latest policy for the policy_id from the policy-engine""" - audit, policy_id, min_version_expected, ignore_policy_names = aud_policy_id - str_metrics = "policy_id({0}), min_version_expected({1}) ignore_policy_names({2})".format( - policy_id, min_version_expected, json.dumps(ignore_policy_names)) + audit, policy_id, expected_versions, ignore_policy_names = aud_policy_id + str_metrics = "policy_id({0}), expected_versions({1}) ignore_policy_names({2})".format( + policy_id, json.dumps(expected_versions), json.dumps(ignore_policy_names)) try: return PolicyRest._get_latest_policy( - audit, policy_id, min_version_expected, ignore_policy_names, str_metrics) + audit, policy_id, expected_versions, ignore_policy_names, str_metrics) except Exception as ex: error_msg = ("{0}: crash {1} {2} at {3}: {4}" @@ -224,19 +224,19 @@ class PolicyRest(object): @staticmethod def _get_latest_policy(audit, policy_id, - min_version_expected, ignore_policy_names, str_metrics): + expected_versions, ignore_policy_names, str_metrics): """retry several times getting the latest policy for the policy_id from the policy-engine""" PolicyRest._lazy_init() latest_policy = None status_code = 0 retry_get_config = audit.kwargs.get("retry_get_config") - expect_policy_removed = (ignore_policy_names and not min_version_expected) + expect_policy_removed = (ignore_policy_names and not expected_versions) for retry in range(1, PolicyRest._policy_retry_count + 1): PolicyRest._logger.debug(str_metrics) done, latest_policy, status_code = PolicyRest._get_latest_policy_once( - audit, policy_id, min_version_expected, ignore_policy_names, + audit, policy_id, expected_versions, ignore_policy_names, expect_policy_removed) if done or not retry_get_config or not PolicyRest._policy_retry_sleep: @@ -271,7 +271,7 @@ class PolicyRest(object): @staticmethod def _get_latest_policy_once(audit, policy_id, - min_version_expected, ignore_policy_names, + expected_versions, ignore_policy_names, expect_policy_removed): """single attempt to get the latest policy for the policy_id from the policy-engine""" @@ -281,7 +281,7 @@ class PolicyRest(object): status_code, policy_id, json.dumps(policy_bodies or [])) latest_policy = PolicyUtils.select_latest_policy( - policy_bodies, min_version_expected, ignore_policy_names + policy_bodies, expected_versions, ignore_policy_names ) if not latest_policy and not expect_policy_removed: @@ -340,12 +340,11 @@ class PolicyRest(object): if not policy: policies_to_find[policy_id] = { POLICY_ID: policy_id, - PolicyRest.MIN_VERSION_EXPECTED: int(policy_version), + PolicyRest.EXPECTED_VERSIONS: {policy_version: True}, PolicyRest.IGNORE_POLICY_NAMES: {} } continue - if int(policy[PolicyRest.MIN_VERSION_EXPECTED]) < int(policy_version): - policy[PolicyRest.MIN_VERSION_EXPECTED] = int(policy_version) + policy[PolicyRest.EXPECTED_VERSIONS][policy_version] = True for (policy_id, policy_names) in policies_removed: if not policy_id: @@ -360,7 +359,7 @@ class PolicyRest(object): policy[PolicyRest.IGNORE_POLICY_NAMES].update(policy_names) apns = [(audit, policy_id, - policy_to_find.get(PolicyRest.MIN_VERSION_EXPECTED), + policy_to_find.get(PolicyRest.EXPECTED_VERSIONS), policy_to_find.get(PolicyRest.IGNORE_POLICY_NAMES)) for (policy_id, policy_to_find) in policies_to_find.items()] @@ -383,7 +382,7 @@ class PolicyRest(object): removed_policies = dict((policy_id, True) for (policy_id, policy_to_find) in policies_to_find.items() - if not policy_to_find.get(PolicyRest.MIN_VERSION_EXPECTED) + if not policy_to_find.get(PolicyRest.EXPECTED_VERSIONS) and policy_to_find.get(PolicyRest.IGNORE_POLICY_NAMES) and policy_id not in updated_policies) diff --git a/policyhandler/policy_updater.py b/policyhandler/policy_updater.py index 3ae8199..7733146 100644 --- a/policyhandler/policy_updater.py +++ b/policyhandler/policy_updater.py @@ -302,20 +302,20 @@ class PolicyUpdater(Thread): else: message = PolicyUpdateMessage(updated_policies, removed_policies, policy_filter_matches, False) - log_updates = ("policies-updated[{0}], removed[{1}]" - .format(len(updated_policies), len(removed_policies))) + log_updates = ("policies-updated[{}], removed[{}], policy_filter_matches[{}]" + .format(len(updated_policies), + len(removed_policies), + len(policy_filter_matches))) audit.reset_http_status_not_found() DeployHandler.policy_update(audit, message) + log_line = "request_id[{}]: {}".format(audit.request_id, str(message)) if not audit.is_success(): result = "- failed to send to deployment-handler {}".format(log_updates) PolicyUpdater._logger.warning(result) else: result = "- sent to deployment-handler {}".format(log_updates) - log_line = "request_id: {} updated_policies: {} removed_policies: {}".format( - audit.request_id, - json.dumps(updated_policies), json.dumps(removed_policies)) audit.audit_done(result=result) PolicyUpdater._logger.info(log_line + " " + result) diff --git a/policyhandler/policy_utils.py b/policyhandler/policy_utils.py index c63f382..da83935 100644 --- a/policyhandler/policy_utils.py +++ b/policyhandler/policy_utils.py @@ -76,7 +76,7 @@ class PolicyUtils(object): return {POLICY_ID:policy_id, POLICY_BODY:policy_body} @staticmethod - def select_latest_policy(policy_bodies, min_version_expected=None, ignore_policy_names=None): + def select_latest_policy(policy_bodies, expected_versions=None, ignore_policy_names=None): """For some reason, the policy-engine returns all version of the policy_bodies. DCAE-Controller is only interested in the latest version """ @@ -88,14 +88,13 @@ class PolicyUtils(object): policy_version = policy_body.get(POLICY_VERSION) if not policy_name or not policy_version or not policy_version.isdigit(): continue - policy_version = int(policy_version) - if min_version_expected and policy_version < min_version_expected: + if expected_versions and policy_version not in expected_versions: continue if ignore_policy_names and policy_name in ignore_policy_names: continue if (not latest_policy_body - or int(latest_policy_body[POLICY_VERSION]) < policy_version): + or int(latest_policy_body[POLICY_VERSION]) < int(policy_version)): latest_policy_body = policy_body return PolicyUtils.parse_policy_config(PolicyUtils.convert_to_policy(latest_policy_body)) diff --git a/pom.xml b/pom.xml index 6af229e..76ae7d9 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property. org.onap.dcaegen2.platform policy-handler dcaegen2-platform-policy-handler - 4.0.0-SNAPSHOT + 4.0.1-SNAPSHOT http://maven.apache.org UTF-8 diff --git a/setup.py b/setup.py index 7505fb3..3667b49 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ from setuptools import setup setup( name='policyhandler', description='DCAE-Controller policy-handler to communicate with policy-engine', - version="4.0.0", + version="4.0.1", author='Alex Shatov', packages=['policyhandler'], zip_safe=False, diff --git a/version.properties b/version.properties index ddff91c..53d98c4 100644 --- a/version.properties +++ b/version.properties @@ -1,6 +1,6 @@ major=4 minor=0 -patch=0 +patch=1 base_version=${major}.${minor}.${patch} release_version=${base_version} snapshot_version=${base_version}-SNAPSHOT -- cgit 1.2.3-korg