From 78ff88f9b3a3d32f941b3b9fedc2abfbaba291cb Mon Sep 17 00:00:00 2001 From: Alex Shatov Date: Thu, 27 Feb 2020 12:45:54 -0500 Subject: 5.1.0 policy-handler - policy-updates from new PDP DCAEGEN2-1851: - policy-handler now supports the policy-update notification from the new policy-engine thru DMaaP MR = no policy-filters - only policy-id values - see README for discoverable config settings of dmaap_mr client = DMaaP MR client has the same flexibility as policy_engine = set the query.timeout to high value like 15000 (default) - requests to DMaaP MR go through a single blocking connection - first catch-up only after draining the policy-updates from DMaaP MR on the first loop - safe parsing of messages from DMaaP MR - policy-engine changed the data type for policy-version field from int to string that is expected to have the semver value - related change to deployment-handler (DCAEGEN2-2085) has to be deployed to handle the non-numeric policyVersion - on new PDP API: http /policy_latest and policy-updates return the new data from the new PDP API with the following fields added/renamed by the policy-handler to keep other policy related parts intact in R4-R6 (see pdp_api/policy_utils.py) * policyName = policy_id + "." + policyVersion.replace(".","-") + ".xml" * policyVersion = str(metadata["policy-version"]) * "config" - is the renamed "properties" from the new PDP API response - enabled the /catch_up and the periodic auto-catch-up for the new PDP API - enabled GET /policies_latest - returns the latest policies for the deployed components - POST /policies_latest - still disabled since no support for the policy-filters is provided for the new PDP API - fixed hiding the Authorization value on comparing the configs - logging of secrets is now sha256 to see whether they changed - added X-ONAP-RequestID to headers the same way as X-ECOMP-RequestID - on policy-update process the removal first, then addition - changed the pool_connections=1 (number of pools) on PDP and DH sides == only a single destination is expected for each - log the exception as fatal into error.log - other minor fixes and refactoring - unit-test coverage 74% - integration testing is requested DCAEGEN2-1976: - policy-handler is enhanced to get user/password from env vars for PDP and DMaaP MR clients and overwriting the Authorization field in https headers received from the discoverable config = to override the Authorization value on policy_engine, set the environment vars $PDP_USER and $PDP_PWD in policy-handler container = to override the Authorization value on dmaap_mr, if using https and user-password authentication, set the environment vars $DMAAP_MR_USER and $DMAAP_MR_PWD in policy-handler container Change-Id: Iad8eab9e20e615a0e0d2822f4735dc64c50aa55c Signed-off-by: Alex Shatov Issue-ID: DCAEGEN2-1851 Issue-ID: DCAEGEN2-1976 --- policyhandler/config.py | 56 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 6 deletions(-) (limited to 'policyhandler/config.py') diff --git a/policyhandler/config.py b/policyhandler/config.py index f8c425a..25ae3a5 100644 --- a/policyhandler/config.py +++ b/policyhandler/config.py @@ -1,5 +1,5 @@ # ================================================================================ -# Copyright (c) 2017-2019 AT&T Intellectual Property. All rights reserved. +# Copyright (c) 2017-2020 AT&T Intellectual Property. All rights reserved. # ================================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ """read and use the config""" +import base64 import copy import json import logging @@ -137,6 +138,7 @@ class Config(object): FIELD_WSERVICE_PORT = "wservice_port" FIELD_TLS = "tls" FIELD_POLICY_ENGINE = "policy_engine" + DMAAP_MR = "dmaap_mr" POOL_CONNECTIONS = "pool_connections" DEPLOY_HANDLER = "deploy_handler" THREAD_POOL_SIZE = "thread_pool_size" @@ -155,6 +157,11 @@ class Config(object): SERVICE_ACTIVATOR = "service_activator" MODE_OF_OPERATION = "mode_of_operation" PDP_API_VERSION = "PDP_API_VERSION" + QUERY_TIMEOUT = "timeout" + PDP_USER = "PDP_USER" + PDP_PWD = "PDP_PWD" + DMAAP_MR_USER = "DMAAP_MR_USER" + DMAAP_MR_PWD = "DMAAP_MR_PWD" system_name = SERVICE_NAME_POLICY_HANDLER wservice_port = 25577 @@ -165,6 +172,8 @@ class Config(object): tls_server_cert_file = None tls_private_key_file = None tls_server_ca_chain_file = None + _pdp_authorization = None + _dmaap_mr_authorization = None _local_config = Settings() discovered_config = Settings() @@ -254,6 +263,18 @@ class Config(object): Config._pdp_api_version = os.environ.get( Config.PDP_API_VERSION, loaded_config.get(Config.PDP_API_VERSION.lower())) + pdp_user = os.environ.get(Config.PDP_USER) + pdp_pwd = os.environ.get(Config.PDP_PWD) + if pdp_user and pdp_pwd: + Config._pdp_authorization = "Basic {}".format(base64.b64encode( + ("{}:{}".format(pdp_user, pdp_pwd)).encode()).decode("utf-8")) + + dmaap_mr_user = os.environ.get(Config.DMAAP_MR_USER) + dmaap_mr_pwd = os.environ.get(Config.DMAAP_MR_PWD) + if dmaap_mr_user and dmaap_mr_pwd: + Config._dmaap_mr_authorization = "Basic {}".format(base64.b64encode( + ("{}:{}".format(dmaap_mr_user, dmaap_mr_pwd)).encode()).decode("utf-8")) + local_config = loaded_config.get(Config.SERVICE_NAME_POLICY_HANDLER, {}) Config.system_name = local_config.get(Config.FIELD_SYSTEM, Config.system_name) @@ -261,6 +282,26 @@ class Config(object): Config._local_config.set_config(local_config, auto_commit=True) + @staticmethod + def _overwrite_discovered_config(audit, discovered_config): + """replace the secrets in discovered_config with data from environment""" + changes = [] + if Config._pdp_authorization: + pdp_cfg = discovered_config.get("policy_engine", {}) + if pdp_cfg.get("url", "").lower().startswith("https:"): + pdp_cfg.get("headers", {})["Authorization"] = Config._pdp_authorization + changes.append("pdp_authorization") + + if Config._dmaap_mr_authorization: + dmaap_mr_cfg = discovered_config.get("dmaap_mr", {}) + if dmaap_mr_cfg.get("url", "").lower().startswith("https:"): + dmaap_mr_cfg.get("headers", {})["Authorization"] = Config._dmaap_mr_authorization + changes.append("dmaap_mr_authorization") + + if changes: + _LOGGER.info(audit.info("overwritten discovered config: {}".format(", ".join(changes)))) + + @staticmethod def discover(audit): """bring the config settings from the discovery service""" @@ -269,14 +310,17 @@ class Config(object): new_config = DiscoveryClient.get_value(audit, discovery_key) if not new_config or not isinstance(new_config, dict): - _LOGGER.warning("unexpected config from discovery: %s", new_config) + _LOGGER.warning(audit.warn("unexpected config from discovery: {}".format(new_config))) return - _LOGGER.debug("loaded config from discovery(%s): %s", - discovery_key, Audit.json_dumps(new_config)) + _LOGGER.debug(audit.debug("loaded config from discovery({}): {}".format( + discovery_key, Audit.json_dumps(new_config)))) + discovered_config = new_config.get(Config.SERVICE_NAME_POLICY_HANDLER) + + Config._overwrite_discovered_config(audit, discovered_config) - Config.discovered_config.set_config(new_config.get(Config.SERVICE_NAME_POLICY_HANDLER)) - _LOGGER.info("config from discovery: %s", Config.discovered_config) + Config.discovered_config.set_config(discovered_config) + _LOGGER.info(audit.info("config from discovery: {}".format(Config.discovered_config))) @staticmethod -- cgit 1.2.3-korg