From 78ff88f9b3a3d32f941b3b9fedc2abfbaba291cb Mon Sep 17 00:00:00 2001 From: Alex Shatov Date: Thu, 27 Feb 2020 12:45:54 -0500 Subject: 5.1.0 policy-handler - policy-updates from new PDP DCAEGEN2-1851: - policy-handler now supports the policy-update notification from the new policy-engine thru DMaaP MR = no policy-filters - only policy-id values - see README for discoverable config settings of dmaap_mr client = DMaaP MR client has the same flexibility as policy_engine = set the query.timeout to high value like 15000 (default) - requests to DMaaP MR go through a single blocking connection - first catch-up only after draining the policy-updates from DMaaP MR on the first loop - safe parsing of messages from DMaaP MR - policy-engine changed the data type for policy-version field from int to string that is expected to have the semver value - related change to deployment-handler (DCAEGEN2-2085) has to be deployed to handle the non-numeric policyVersion - on new PDP API: http /policy_latest and policy-updates return the new data from the new PDP API with the following fields added/renamed by the policy-handler to keep other policy related parts intact in R4-R6 (see pdp_api/policy_utils.py) * policyName = policy_id + "." + policyVersion.replace(".","-") + ".xml" * policyVersion = str(metadata["policy-version"]) * "config" - is the renamed "properties" from the new PDP API response - enabled the /catch_up and the periodic auto-catch-up for the new PDP API - enabled GET /policies_latest - returns the latest policies for the deployed components - POST /policies_latest - still disabled since no support for the policy-filters is provided for the new PDP API - fixed hiding the Authorization value on comparing the configs - logging of secrets is now sha256 to see whether they changed - added X-ONAP-RequestID to headers the same way as X-ECOMP-RequestID - on policy-update process the removal first, then addition - changed the pool_connections=1 (number of pools) on PDP and DH sides == only a single destination is expected for each - log the exception as fatal into error.log - other minor fixes and refactoring - unit-test coverage 74% - integration testing is requested DCAEGEN2-1976: - policy-handler is enhanced to get user/password from env vars for PDP and DMaaP MR clients and overwriting the Authorization field in https headers received from the discoverable config = to override the Authorization value on policy_engine, set the environment vars $PDP_USER and $PDP_PWD in policy-handler container = to override the Authorization value on dmaap_mr, if using https and user-password authentication, set the environment vars $DMAAP_MR_USER and $DMAAP_MR_PWD in policy-handler container Change-Id: Iad8eab9e20e615a0e0d2822f4735dc64c50aa55c Signed-off-by: Alex Shatov Issue-ID: DCAEGEN2-1851 Issue-ID: DCAEGEN2-1976 --- policyhandler/pdp_api/policy_matcher.py | 93 ++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) (limited to 'policyhandler/pdp_api/policy_matcher.py') diff --git a/policyhandler/pdp_api/policy_matcher.py b/policyhandler/pdp_api/policy_matcher.py index 57258c3..2972fb8 100644 --- a/policyhandler/pdp_api/policy_matcher.py +++ b/policyhandler/pdp_api/policy_matcher.py @@ -1,5 +1,5 @@ # ================================================================================ -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,7 +19,98 @@ import os +from ..deploy_handler import DeployHandler, PolicyUpdateMessage +from ..policy_consts import (ERRORED_POLICIES, LATEST_POLICIES, POLICY_BODY, + POLICY_VERSIONS) +from ..utils import Utils +from .pdp_consts import POLICY_VERSION +from .policy_rest import PolicyRest + +_LOGGER = Utils.get_logger(__file__) class PolicyMatcher(object): """policy-matcher - static class""" + PENDING_UPDATE = "pending_update" PDP_API_FOLDER = os.path.basename(os.path.dirname(os.path.realpath(__file__))) + + @staticmethod + def build_catch_up_message(audit, deployed_policies, _=None): + """find the latest policies from policy-engine for the deployed policies""" + + if not deployed_policies: + error_txt = "no deployed policies" + _LOGGER.warning(error_txt) + return {"error": error_txt}, None + + pdp_response = PolicyRest.get_latest_policies(audit, policy_ids=list(deployed_policies)) + + if not audit.is_success(): + error_txt = "failed to retrieve policies from policy-engine" + _LOGGER.warning(error_txt) + return {"error": error_txt}, None + + latest_policies = pdp_response.get(LATEST_POLICIES, {}) + errored_policies = pdp_response.get(ERRORED_POLICIES, {}) + + latest_policies, changed_policies = PolicyMatcher._match_policies( + latest_policies, deployed_policies) + + errored_policies = dict((policy_id, policy) + for (policy_id, policy) in errored_policies.items() + if deployed_policies.get(policy_id, {}).get(POLICY_VERSIONS)) + + removed_policies = dict( + (policy_id, True) + for (policy_id, deployed_policy) in deployed_policies.items() + if deployed_policy.get(POLICY_VERSIONS) + and policy_id not in latest_policies + and policy_id not in errored_policies + ) + + return ({LATEST_POLICIES: latest_policies, ERRORED_POLICIES: errored_policies}, + PolicyUpdateMessage(changed_policies, removed_policies)) + + @staticmethod + def match_to_deployed_policies(audit, policies_updated, policies_removed): + """match the policies_updated, policies_removed versus deployed policies""" + _, deployed_policies, _ = DeployHandler.get_deployed_policies(audit) + if not audit.is_success(): + return {}, {}, {} + + _, changed_policies = PolicyMatcher._match_policies(policies_updated, deployed_policies) + + policies_removed = dict((policy_id, policy) + for (policy_id, policy) in policies_removed.items() + if deployed_policies.get(policy_id, {}).get(POLICY_VERSIONS)) + + return changed_policies, policies_removed, {} + + + @staticmethod + def _match_policies(policies, deployed_policies): + """ + Match policies to deployed policies by policy_id. + + Also calculates the policies that changed in comparison to deployed policies + """ + matching_policies = {} + changed_policies = {} + + policies = policies or {} + deployed_policies = deployed_policies or {} + + for (policy_id, policy) in policies.items(): + new_version = policy.get(POLICY_BODY, {}).get(POLICY_VERSION) + deployed_policy = deployed_policies.get(policy_id) + + if deployed_policy: + matching_policies[policy_id] = policy + + policy_changed = (deployed_policy and new_version + and (deployed_policy.get(PolicyMatcher.PENDING_UPDATE) + or {new_version} ^ + deployed_policy.get(POLICY_VERSIONS, {}).keys())) + if policy_changed: + changed_policies[policy_id] = policy + + return matching_policies, changed_policies -- cgit 1.2.3-korg