From 1d693376205c66af93283d04e8e9740c947a7d02 Mon Sep 17 00:00:00 2001 From: Alex Shatov Date: Fri, 24 Aug 2018 13:15:04 -0400 Subject: 4.2.0 policy-handler - periodic reconfigure - reconfigure == periodically retrieve the policy-handler config from consul-kv and compare to previous config and subconfigs. If changed, reconfigure the subunits - selectively change one or any settings for the following = catch_up timer interval = reconfigure timer interval = deployment-handler url and params (thread-safe) = policy-engine url and params (thread-safe) = web-socket url to policy-engine (through a callback) - each subunit has its own Settings that keep track of changes - try-catch and metrics around discovery - consul API - hidden the secrets from logs - froze the web-socket version to 0.49.0 because 0.50.0 and 0.51.0 are broken - looking around for stable alternatives - fixed-adapted the callbacks passed to the web-socket lib that changed its API in 0.49.0 and later - log the stack on the exception occurring in the web-socket lib - unit test refactoring Change-Id: Id53bad59660a197f59d9aeb7c05ab761d1060cd0 Signed-off-by: Alex Shatov Issue-ID: DCAEGEN2-470 --- tests/mock_config.json | 36 +++++++++++++++ tests/mock_settings.py | 79 ++++++++++++++++++++++++++++----- tests/test_policy_utils.py | 2 +- tests/test_policyhandler.py | 105 ++++++++++++++++++-------------------------- tests/test_step_timer.py | 2 +- 5 files changed, 149 insertions(+), 75 deletions(-) create mode 100644 tests/mock_config.json (limited to 'tests') diff --git a/tests/mock_config.json b/tests/mock_config.json new file mode 100644 index 0000000..98b0d19 --- /dev/null +++ b/tests/mock_config.json @@ -0,0 +1,36 @@ +{ + "policy_handler" : { + "system" : "policy_handler", + "thread_pool_size" : 4, + "pool_connections" : 20, + "policy_retry_count" : 5, + "policy_retry_sleep" : 5, + "catch_up" : { + "interval" : 10 + }, + "reconfigure" : { + "interval" : 10 + }, + "policy_engine" : { + "url" : "https://pdp-server:8081", + "path_pdp" : "/pdp/", + "path_api" : "/pdp/api/", + "headers" : { + "Accept" : "application/json", + "Content-Type" : "application/json", + "ClientAuth" : "Basic user", + "Authorization" : "Basic auth", + "Environment" : "TEST" + }, + "target_entity" : "policy_engine" + }, + "deploy_handler" : { + "target_entity" : "deployment_handler", + "url" : "http://deployment_handler:8188", + "max_msg_length_mb" : 5, + "query" : { + "cfy_tenant_name" : "default_tenant" + } + } + } +} diff --git a/tests/mock_settings.py b/tests/mock_settings.py index 017ad7e..7e05ecf 100644 --- a/tests/mock_settings.py +++ b/tests/mock_settings.py @@ -17,43 +17,100 @@ # ECOMP is a trademark and service mark of AT&T Intellectual Property. """settings that are general to all tests""" +import copy import json import logging import sys import uuid from datetime import datetime +from functools import wraps from policyhandler import LogWriter from policyhandler.config import Config +from policyhandler.discovery import DiscoveryClient from policyhandler.onap.audit import Audit +class MonkeyHttpResponse(object): + """Monkey http reposne""" + def __init__(self, headers): + self.headers = headers or {} + + +class MonkeyedResponse(object): + """Monkey response""" + def __init__(self, full_path, res_json, json_body=None, headers=None): + self.full_path = full_path + self.req_json = json_body or {} + self.status_code = 200 + self.request = MonkeyHttpResponse(headers) + self.res = res_json + self.text = json.dumps(self.res) + + def json(self): + """returns json of response""" + return self.res + + def raise_for_status(self): + """ignoring""" + pass + + +def _fix_discover_config(func): + """the decorator""" + if not func: + return None + + def mocked_discover_get_value(*_): + """monkeypatch for get from consul""" + return copy.deepcopy(Settings.mock_config) + + @wraps(func) + def wrapper(*args, **kwargs): + """override the DiscoveryClient.get_value to fake discovering the config""" + + old_get_value = DiscoveryClient.get_value + DiscoveryClient.get_value = mocked_discover_get_value + + func_result = func(*args, **kwargs) + + DiscoveryClient.get_value = old_get_value + + return func_result + return wrapper + class Settings(object): """init all locals""" + _loaded = False logger = None RUN_TS = datetime.utcnow().isoformat()[:-3] + 'Z' - dicovered_config = None + mock_config = None deploy_handler_instance_uuid = str(uuid.uuid4()) @staticmethod + @_fix_discover_config def init(): """init configs""" - Config.load_from_file() - - with open("etc_upload/config.json", 'r') as config_json: - Settings.dicovered_config = json.load(config_json) + if Settings._loaded: + return + Settings._loaded = True - Config.load_from_file("etc_upload/config.json") + Config.init_config() - Config.settings["catch_up"] = {"interval": 10} + with open("tests/mock_config.json", 'r') as config_json: + Settings.mock_config = json.load(config_json) Settings.logger = logging.getLogger("policy_handler.unit_test") sys.stdout = LogWriter(Settings.logger.info) sys.stderr = LogWriter(Settings.logger.error) - print("print ========== run_policy_handler ==========") + print("print is expected to be in the log") Settings.logger.info("========== run_policy_handler ==========") - Audit.init(Config.get_system_name(), Config.LOGGER_CONFIG_FILE_PATH) + Audit.init(Config.system_name, Config.LOGGER_CONFIG_FILE_PATH) + audit = Audit(req_message="start testing policy handler") + + Config.discover(audit) + + Settings.logger.info("testing policy_handler with config: %s", Config.discovered_config) - Settings.logger.info("starting policy_handler with config:") - Settings.logger.info(Audit.log_json_dumps(Config.settings)) + audit.audit_done(" -- started") diff --git a/tests/test_policy_utils.py b/tests/test_policy_utils.py index b88f1ea..dcf6ccb 100644 --- a/tests/test_policy_utils.py +++ b/tests/test_policy_utils.py @@ -25,7 +25,7 @@ import re from policyhandler.config import Config from policyhandler.policy_utils import RegexCoarser -Config.load_from_file() +Config.init_config() LOGGER = logging.getLogger("policy_handler.unit_test_policy_utils") diff --git a/tests/test_policyhandler.py b/tests/test_policyhandler.py index dec760a..d14aeea 100644 --- a/tests/test_policyhandler.py +++ b/tests/test_policyhandler.py @@ -18,16 +18,18 @@ """test of the package for policy-handler of DCAE-Controller""" +import base64 import copy import json import re import time import uuid -import pytest import cherrypy from cherrypy.test.helper import CPWebCase +import pytest + from policyhandler.config import Config from policyhandler.deploy_handler import DeployHandler from policyhandler.discovery import DiscoveryClient @@ -42,57 +44,10 @@ from policyhandler.policy_rest import PolicyRest from policyhandler.policy_utils import PolicyUtils, Utils from policyhandler.web_server import _PolicyWeb -from .mock_settings import Settings +from .mock_settings import MonkeyedResponse, Settings Settings.init() -class MonkeyHttpResponse(object): - """Monkey http reposne""" - def __init__(self, headers): - self.headers = headers or {} - - -class MonkeyedResponse(object): - """Monkey response""" - def __init__(self, full_path, res_json, json_body=None, headers=None): - self.full_path = full_path - self.req_json = json_body or {} - self.status_code = 200 - self.request = MonkeyHttpResponse(headers) - self.res = res_json - self.text = json.dumps(self.res) - - def json(self): - """returns json of response""" - return self.res - - def raise_for_status(self): - """ignoring""" - pass - - -def monkeyed_discovery(full_path): - """monkeypatch for get from consul""" - res_json = {} - if full_path == DiscoveryClient.CONSUL_SERVICE_MASK.format(Config.settings["deploy_handler"]): - res_json = [{ - "ServiceAddress": "1.1.1.1", - "ServicePort": "123" - }] - elif full_path == DiscoveryClient.CONSUL_KV_MASK.format(Config.get_system_name()): - res_json = copy.deepcopy(Settings.dicovered_config) - return MonkeyedResponse(full_path, res_json) - - -@pytest.fixture() -def fix_discovery(monkeypatch): - """monkeyed discovery request.get""" - Settings.logger.info("setup fix_discovery") - monkeypatch.setattr('policyhandler.discovery.requests.get', monkeyed_discovery) - yield fix_discovery # provide the fixture value - Settings.logger.info("teardown fix_discovery") - - class MonkeyPolicyBody(object): """policy body that policy-engine returns""" @staticmethod @@ -190,7 +145,6 @@ class MockPolicyEngine(object): for k, v in MockPolicyEngine.gen_all_policies_latest().items() if re.match(match_to_policy_name, k)) - MockPolicyEngine.init() @@ -289,9 +243,30 @@ def fix_select_latest_policies_boom(monkeypatch): yield fix_select_latest_policies_boom Settings.logger.info("teardown fix_select_latest_policies_boom") +@pytest.fixture() +def fix_discovery(monkeypatch): + """monkeyed discovery request.get""" + def monkeyed_discovery(full_path): + """monkeypatch for get from consul""" + res_json = {} + if full_path == DiscoveryClient.CONSUL_SERVICE_MASK.format( + Config.discovered_config.get_by_key(Config.DEPLOY_HANDLER)): + res_json = [{ + "ServiceAddress": "1.1.1.1", + "ServicePort": "123" + }] + elif full_path == DiscoveryClient.CONSUL_KV_MASK.format(Config.system_name): + res_json = [{"Value": base64.b64encode( + json.dumps(Settings.mock_config).encode()).decode("utf-8")}] + return MonkeyedResponse(full_path, res_json) + + Settings.logger.info("setup fix_discovery") + monkeypatch.setattr('policyhandler.discovery.requests.get', monkeyed_discovery) + yield fix_discovery # provide the fixture value + Settings.logger.info("teardown fix_discovery") @pytest.fixture() -def fix_deploy_handler(monkeypatch, fix_discovery): +def fix_deploy_handler(monkeypatch): """monkeyed requests to deployment-handler""" def monkeyed_deploy_handler_put(full_path, json=None, headers=None, params=None): """monkeypatch for policy-update request.put to deploy_handler""" @@ -313,11 +288,12 @@ def fix_deploy_handler(monkeypatch, fix_discovery): monkeyed_deploy_handler_get) yield fix_deploy_handler # provide the fixture value + audit.audit_done("teardown") Settings.logger.info("teardown fix_deploy_handler") @pytest.fixture() -def fix_deploy_handler_fail(monkeypatch, fix_discovery): +def fix_deploy_handler_fail(monkeypatch): """monkeyed failed discovery request.get""" def monkeyed_deploy_handler_put(full_path, json=None, headers=None, params=None): """monkeypatch for deploy_handler""" @@ -335,16 +311,16 @@ def fix_deploy_handler_fail(monkeypatch, fix_discovery): None, headers) @staticmethod - def monkeyed_deploy_handler_init(audit_ignore, rediscover=False): + def monkeyed_deploy_handler_init(audit_ignore): """monkeypatch for deploy_handler init""" DeployHandler._url = None Settings.logger.info("setup fix_deploy_handler_fail") - config_catch_up = Config.settings["catch_up"] - Config.settings["catch_up"] = {"interval": 1} + config_catch_up = Config.discovered_config.get_by_key("catch_up") + Config.discovered_config.update("catch_up", {"interval": 1}) audit = Audit(req_message="fix_deploy_handler_fail") - DeployHandler._lazy_init(audit, rediscover=True) + DeployHandler._lazy_init(audit) monkeypatch.setattr('policyhandler.deploy_handler.DeployHandler._lazy_init', monkeyed_deploy_handler_init) @@ -354,8 +330,9 @@ def fix_deploy_handler_fail(monkeypatch, fix_discovery): monkeyed_deploy_handler_get) yield fix_deploy_handler_fail + audit.audit_done("teardown") Settings.logger.info("teardown fix_deploy_handler_fail") - Config.settings["catch_up"] = config_catch_up + Config.discovered_config.update("catch_up", config_catch_up) @pytest.fixture() @@ -438,7 +415,7 @@ def fix_policy_receiver_websocket(monkeypatch): Settings.logger.info("teardown fix_policy_receiver_websocket") -def test_get_policy_latest(fix_pdp_post): +def test_get_policy_latest(fix_pdp_post, fix_discovery): """test /policy_latest/""" policy_id, expected_policy = MockPolicyEngine.gen_policy_latest(3) @@ -453,7 +430,7 @@ def test_get_policy_latest(fix_pdp_post): -@pytest.mark.usefixtures("fix_pdp_post") +@pytest.mark.usefixtures("fix_pdp_post", "fix_discovery") class WebServerTest(CPWebCase): """testing the web-server - runs tests in alphabetical order of method names""" def setup_server(): @@ -630,6 +607,7 @@ class WebServerTest(CPWebCase): WebServerTest.do_gc_test = False Settings.logger.info("shutdown...") + audit.audit_done("shutdown") result = self.getPage("/shutdown") Settings.logger.info("shutdown result: %s", result) self.assertStatus('200 OK') @@ -637,7 +615,7 @@ class WebServerTest(CPWebCase): time.sleep(1) -@pytest.mark.usefixtures("fix_pdp_post_boom") +@pytest.mark.usefixtures("fix_pdp_post_boom", "fix_discovery") class WebServerPDPBoomTest(CPWebCase): """testing the web-server - runs tests in alphabetical order of method names""" def setup_server(): @@ -791,6 +769,7 @@ class WebServerPDPBoomTest(CPWebCase): WebServerPDPBoomTest.do_gc_test = False Settings.logger.info("shutdown...") + audit.audit_done("shutdown") result = self.getPage("/shutdown") Settings.logger.info("shutdown result: %s", result) self.assertStatus('200 OK') @@ -798,7 +777,7 @@ class WebServerPDPBoomTest(CPWebCase): time.sleep(1) -@pytest.mark.usefixtures("fix_pdp_post", "fix_select_latest_policies_boom") +@pytest.mark.usefixtures("fix_pdp_post", "fix_select_latest_policies_boom", "fix_discovery") class WebServerInternalBoomTest(CPWebCase): """testing the web-server - runs tests in alphabetical order of method names""" def setup_server(): @@ -952,6 +931,7 @@ class WebServerInternalBoomTest(CPWebCase): WebServerInternalBoomTest.do_gc_test = False Settings.logger.info("shutdown...") + audit.audit_done("shutdown") result = self.getPage("/shutdown") Settings.logger.info("shutdown result: %s", result) self.assertStatus('200 OK') @@ -962,7 +942,8 @@ class WebServerInternalBoomTest(CPWebCase): @pytest.mark.usefixtures( "fix_pdp_post_big", "fix_deploy_handler_fail", - "fix_policy_receiver_websocket" + "fix_policy_receiver_websocket", + "fix_discovery" ) def test_catch_ups_failed_dh(): """test run policy handler with catchups and failed deployment-handler""" diff --git a/tests/test_step_timer.py b/tests/test_step_timer.py index ff89388..d4a0df1 100644 --- a/tests/test_step_timer.py +++ b/tests/test_step_timer.py @@ -26,7 +26,7 @@ from datetime import datetime from policyhandler.config import Config from policyhandler.step_timer import StepTimer -Config.load_from_file() +Config.init_config() class MockTimerController(object): -- cgit 1.2.3-korg