From 78ff88f9b3a3d32f941b3b9fedc2abfbaba291cb Mon Sep 17 00:00:00 2001 From: Alex Shatov Date: Thu, 27 Feb 2020 12:45:54 -0500 Subject: 5.1.0 policy-handler - policy-updates from new PDP DCAEGEN2-1851: - policy-handler now supports the policy-update notification from the new policy-engine thru DMaaP MR = no policy-filters - only policy-id values - see README for discoverable config settings of dmaap_mr client = DMaaP MR client has the same flexibility as policy_engine = set the query.timeout to high value like 15000 (default) - requests to DMaaP MR go through a single blocking connection - first catch-up only after draining the policy-updates from DMaaP MR on the first loop - safe parsing of messages from DMaaP MR - policy-engine changed the data type for policy-version field from int to string that is expected to have the semver value - related change to deployment-handler (DCAEGEN2-2085) has to be deployed to handle the non-numeric policyVersion - on new PDP API: http /policy_latest and policy-updates return the new data from the new PDP API with the following fields added/renamed by the policy-handler to keep other policy related parts intact in R4-R6 (see pdp_api/policy_utils.py) * policyName = policy_id + "." + policyVersion.replace(".","-") + ".xml" * policyVersion = str(metadata["policy-version"]) * "config" - is the renamed "properties" from the new PDP API response - enabled the /catch_up and the periodic auto-catch-up for the new PDP API - enabled GET /policies_latest - returns the latest policies for the deployed components - POST /policies_latest - still disabled since no support for the policy-filters is provided for the new PDP API - fixed hiding the Authorization value on comparing the configs - logging of secrets is now sha256 to see whether they changed - added X-ONAP-RequestID to headers the same way as X-ECOMP-RequestID - on policy-update process the removal first, then addition - changed the pool_connections=1 (number of pools) on PDP and DH sides == only a single destination is expected for each - log the exception as fatal into error.log - other minor fixes and refactoring - unit-test coverage 74% - integration testing is requested DCAEGEN2-1976: - policy-handler is enhanced to get user/password from env vars for PDP and DMaaP MR clients and overwriting the Authorization field in https headers received from the discoverable config = to override the Authorization value on policy_engine, set the environment vars $PDP_USER and $PDP_PWD in policy-handler container = to override the Authorization value on dmaap_mr, if using https and user-password authentication, set the environment vars $DMAAP_MR_USER and $DMAAP_MR_PWD in policy-handler container Change-Id: Iad8eab9e20e615a0e0d2822f4735dc64c50aa55c Signed-off-by: Alex Shatov Issue-ID: DCAEGEN2-1851 Issue-ID: DCAEGEN2-1976 --- policyhandler/pdp_api/policy_updates.py | 87 +++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 10 deletions(-) (limited to 'policyhandler/pdp_api/policy_updates.py') diff --git a/policyhandler/pdp_api/policy_updates.py b/policyhandler/pdp_api/policy_updates.py index eb3c3d1..15f5b0a 100644 --- a/policyhandler/pdp_api/policy_updates.py +++ b/policyhandler/pdp_api/policy_updates.py @@ -1,5 +1,5 @@ # ================================================================================ -# Copyright (c) 2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2019-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,10 +17,13 @@ """policy-updates accumulates the policy-update notifications from PDP""" +import json import os -from ..utils import Utils, ToBeImplementedException - +from ..policy_consts import POLICY_BODY, POLICY_ID, POLICY_NAMES +from ..utils import Utils +from .pdp_consts import DEPLOYED_POLICIES, POLICY_NAME, UNDEPLOYED_POLICIES +from .policy_utils import PolicyUtils _LOGGER = Utils.get_logger(__file__) @@ -30,9 +33,12 @@ class PolicyUpdates(object): def __init__(self): """init and reset""" + self._audit = None + self._policies_updated = {} + self._policies_removed = {} def reset(self): - """resets the state""" + """resets the state - removes the pending policy-updates""" self.__init__() def pop_policy_updates(self): @@ -40,10 +46,71 @@ class PolicyUpdates(object): Returns the consolidated (audit, policies_updated, policies_removed) and resets the state """ - _LOGGER.info("to_be_implemented") - return None, None, None + if not self._audit: + return None, None, None + + audit = self._audit + policies_updated = self._policies_updated + policies_removed = self._policies_removed + + self.reset() + + return audit, policies_updated, policies_removed + + + def push_policy_updates(self, audit, multi_policies_updated): + """ + consolidate the new policies_updated, policies_removed to existing ones + + receives + :multi_policies_updated: as [ + {DEPLOYED_POLICIES: [{PDP_METADATA: {POLICY_ID: , + POLICY_VERSION: }}, ...], + UNDEPLOYED_POLICIES: [{PDP_METADATA: {POLICY_ID: , + POLICY_VERSION: }}, ...] + }, ...] + """ + for p_single_updated in multi_policies_updated: + for p_undeployed in p_single_updated.get(UNDEPLOYED_POLICIES, []): + policy = PolicyUtils.convert_to_policy(p_undeployed) + if not policy: + continue + policy_id = policy.get(POLICY_ID) + policy_name = policy.get(POLICY_BODY, {}).get(POLICY_NAME) + + if policy_id in self._policies_removed: + policy = self._policies_removed[policy_id] + + if POLICY_NAMES not in policy: + policy[POLICY_NAMES] = {} + policy[POLICY_NAMES][policy_name] = True + self._policies_removed[policy_id] = policy + + for p_deployed in p_single_updated.get(DEPLOYED_POLICIES, []): + policy = PolicyUtils.convert_to_policy(p_deployed) + if not policy: + continue + policy_id = policy.get(POLICY_ID) + policy_name = policy.get(POLICY_BODY, {}).get(POLICY_NAME) + + self._policies_updated[policy_id] = policy + + rm_policy_names = self._policies_removed.get(policy_id, {}).get(POLICY_NAMES) + if rm_policy_names and policy_name in rm_policy_names: + del rm_policy_names[policy_name] + + req_message = ("policy-update notification - updated[{}], removed[{}]" + .format(len(self._policies_updated), + len(self._policies_removed))) + + if not self._audit: + self._audit = audit + else: + audit.audit_done(result="policy-updates queued to request_id({})" + .format(self._audit.request_id)) + self._audit.req_message = req_message - def push_policy_updates(self, *_): - """consolidate the new policies_updated, policies_removed to existing ones""" - _LOGGER.info("to_be_implemented") - raise ToBeImplementedException() + _LOGGER.info( + "pending(%s) for %s policies_updated %s policies_removed %s", + self._audit.request_id, req_message, + json.dumps(self._policies_updated), json.dumps(self._policies_removed)) -- cgit 1.2.3-korg