From 9cbe049d92690f94ca4e07e6a823b90340b922c1 Mon Sep 17 00:00:00 2001 From: Alex Shatov Date: Wed, 10 Jan 2018 11:27:44 -0500 Subject: variable collection of policies per component * new feature variable collection of policies per component in DCAE * Unit Test coverage 78% * moved module docstring below the license text Change-Id: Iefe6d4c31e2e125194781edc79e69af2c11e96ef Issue-ID: DCAEGEN2-249 Signed-off-by: Alex Shatov --- dcae-policy/dcaepolicy-node-type.yaml | 44 ++++++++++- dcae-policy/dcaepolicyplugin/__init__.py | 3 +- dcae-policy/dcaepolicyplugin/discovery.py | 4 +- dcae-policy/dcaepolicyplugin/tasks.py | 127 ++++++++++++++++++++++++------ dcae-policy/pom.xml | 2 +- dcae-policy/setup.py | 4 +- dcae-policy/tests/__init__.py | 18 +++++ dcae-policy/tests/log_ctx.py | 12 +-- dcae-policy/tests/mock_cloudify_ctx.py | 3 +- dcae-policy/tests/test_tasks.py | 106 +++++++++++++++++++------ dcae-policy/tox-local.ini | 3 +- 11 files changed, 266 insertions(+), 60 deletions(-) create mode 100644 dcae-policy/tests/__init__.py diff --git a/dcae-policy/dcaepolicy-node-type.yaml b/dcae-policy/dcaepolicy-node-type.yaml index 515d6b9..b9d8a66 100644 --- a/dcae-policy/dcaepolicy-node-type.yaml +++ b/dcae-policy/dcaepolicy-node-type.yaml @@ -27,20 +27,58 @@ plugins: dcaepolicy: executor: 'central_deployment_agent' package_name: dcaepolicyplugin - package_version: 1.0.0 + package_version: 2.0.0 + +data_types: + # the properties inside dcae.data.policy_filter are identical to /getConfig API of policy-engine except the requestID field. + # refer to policy-engine /getConfig wiki for explanation of these properties. + # policy-engine /getConfig wiki: The filter works as a combined "AND" operation. + # To retrieve all policies using "sample" as configName, + # the request needs to have policyName = ".*" and configName as = "sample" + # configAttributes is a key-value dictionary + dcae.data.policy_filter: + properties: + policyName: + type: string + default: "DCAE.Config_.*" + configName: + type: string + default: "" + onapName: + type: string + default: "DCAE" + configAttributes: + default: {} + unique: + type: boolean + default: false node_types: + # node that points to a single latest policy identified by policy_id + # policy_id is the versionless left part of policyName in policy-engine dcae.nodes.policy: derived_from: cloudify.nodes.Root properties: policy_id: - description: PK to policy in policy-engine + description: versionless key to policy in policy-engine type: string default: DCAE.Config_unknown-policy policy_required: description: whether to throw an exception when failed to get the policy type: boolean - default: true + default: false + interfaces: + cloudify.interfaces.lifecycle: + create: + implementation: dcaepolicy.dcaepolicyplugin.policy_get + + # node that points to varying collection of policies by selection criteria = policy_filter. + dcae.nodes.policies: + derived_from: cloudify.nodes.Root + properties: + policy_filter: + type: dcae.data.policy_filter + default: {} interfaces: cloudify.interfaces.lifecycle: create: diff --git a/dcae-policy/dcaepolicyplugin/__init__.py b/dcae-policy/dcaepolicyplugin/__init__.py index d2946a6..4b64484 100644 --- a/dcae-policy/dcaepolicyplugin/__init__.py +++ b/dcae-policy/dcaepolicyplugin/__init__.py @@ -1,4 +1,3 @@ -""":policyplugin: gets the policy from policy-handler and stores it into runtime properties""" # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ @@ -19,4 +18,6 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. +""":policyplugin: gets the policy from policy-handler and stores it into runtime properties""" + from .tasks import policy_get diff --git a/dcae-policy/dcaepolicyplugin/discovery.py b/dcae-policy/dcaepolicyplugin/discovery.py index 8cdbde1..6bed180 100644 --- a/dcae-policy/dcaepolicyplugin/discovery.py +++ b/dcae-policy/dcaepolicyplugin/discovery.py @@ -1,5 +1,3 @@ -"""client to talk to consul on standard port 8500""" - # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ @@ -20,6 +18,8 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. +"""client to talk to consul on standard port 8500""" + import requests # it is safe to assume that consul agent is at localhost:8500 along with cloudify manager diff --git a/dcae-policy/dcaepolicyplugin/tasks.py b/dcae-policy/dcaepolicyplugin/tasks.py index 2676864..fb98412 100644 --- a/dcae-policy/dcaepolicyplugin/tasks.py +++ b/dcae-policy/dcaepolicyplugin/tasks.py @@ -18,11 +18,12 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. -# Lifecycle interface calls for DockerContainer +"""tasks are the cloudify operations invoked on interfaces defined in the blueprint""" import json import uuid - +import copy +import traceback import requests from cloudify import ctx @@ -35,12 +36,20 @@ from .discovery import discover_service_url POLICY_ID = 'policy_id' POLICY_REQUIRED = 'policy_required' POLICY_BODY = 'policy_body' +POLICIES_FILTERED = 'policies_filtered' +POLICY_FILTER = 'policy_filter' + +REQUEST_ID = "requestID" + DCAE_POLICY_TYPE = 'dcae.nodes.policy' +DCAE_POLICIES_TYPE = 'dcae.nodes.policies' +DCAE_POLICY_TYPES = [DCAE_POLICY_TYPE, DCAE_POLICIES_TYPE] class PolicyHandler(object): """talk to policy-handler""" SERVICE_NAME_POLICY_HANDLER = "policy_handler" X_ECOMP_REQUESTID = 'X-ECOMP-RequestID' + STATUS_CODE_POLICIES_NOT_FOUND = 404 _url = None @staticmethod @@ -49,53 +58,127 @@ class PolicyHandler(object): if PolicyHandler._url: return - PolicyHandler._url = "{0}/policy_latest".format( - discover_service_url(PolicyHandler.SERVICE_NAME_POLICY_HANDLER) - ) + PolicyHandler._url = discover_service_url(PolicyHandler.SERVICE_NAME_POLICY_HANDLER) @staticmethod def get_latest_policy(policy_id): """retrieve the latest policy for policy_id from policy-handler""" PolicyHandler._lazy_init() - ph_path = "{0}/{1}".format(PolicyHandler._url, policy_id) + ph_path = "{0}/policy_latest/{1}".format(PolicyHandler._url, policy_id) headers = {PolicyHandler.X_ECOMP_REQUESTID: str(uuid.uuid4())} ctx.logger.info("getting latest policy from {0} headers={1}".format( \ ph_path, json.dumps(headers))) res = requests.get(ph_path, headers=headers) + + if res.status_code == PolicyHandler.STATUS_CODE_POLICIES_NOT_FOUND: + return + res.raise_for_status() + return res.json() - if res.status_code == requests.codes.ok: - return res.json() - return {} + @staticmethod + def find_latest_policies(policy_filter): + """retrieve the latest policies by policy filter (selection criteria) from policy-handler""" + PolicyHandler._lazy_init() -######################################################### -@operation -def policy_get(**kwargs): - """retrieve the latest policy_body for policy_id property and save it in runtime_properties""" - if ctx.type != NODE_INSTANCE or DCAE_POLICY_TYPE not in ctx.node.type_hierarchy: - error = "can only invoke policy_get on node of type {0}".format(DCAE_POLICY_TYPE) - ctx.logger.error(error) - raise NonRecoverableError(error) + ph_path = "{0}/policies_latest".format(PolicyHandler._url) + headers = { + PolicyHandler.X_ECOMP_REQUESTID: policy_filter.get(REQUEST_ID, str(uuid.uuid4())) + } + + ctx.logger.info("finding the latest polices from {0} by {1} headers={2}".format( \ + ph_path, json.dumps(policy_filter), json.dumps(headers))) + + res = requests.post(ph_path, json=policy_filter, headers=headers) - if POLICY_ID not in ctx.node.properties: + if res.status_code == PolicyHandler.STATUS_CODE_POLICIES_NOT_FOUND: + return + + res.raise_for_status() + return res.json() + +def _policy_get(): + """ + dcae.nodes.policy - + retrieve the latest policy_body for policy_id property + and save policy_body in runtime_properties + """ + if DCAE_POLICY_TYPE not in ctx.node.type_hierarchy: + return + + policy_id = ctx.node.properties.get(POLICY_ID) + if not policy_id: error = "no {0} found in ctx.node.properties".format(POLICY_ID) ctx.logger.error(error) raise NonRecoverableError(error) try: - policy_id = ctx.node.properties[POLICY_ID] policy = PolicyHandler.get_latest_policy(policy_id) if not policy: raise NonRecoverableError("policy not found for policy_id {0}".format(policy_id)) - ctx.logger.info("found policy {0}".format(json.dumps(policy))) + ctx.logger.info("found policy {0}: {1}".format(policy_id, json.dumps(policy))) if POLICY_BODY in policy: ctx.instance.runtime_properties[POLICY_BODY] = policy[POLICY_BODY] except Exception as ex: - error = "failed to get policy: {0}".format(str(ex)) - ctx.logger.error(error) + error = "failed to get policy({0}): {1}".format(policy_id, str(ex)) + ctx.logger.error("{0}: {1}".format(error, traceback.format_exc())) if ctx.node.properties.get(POLICY_REQUIRED, True): raise NonRecoverableError(error) + + return True + +def _policies_find(): + """ + dcae.nodes.policies - + retrieve the latest policies for selection criteria + and save found policies in runtime_properties + """ + if DCAE_POLICIES_TYPE not in ctx.node.type_hierarchy: + return + + try: + policy_filter = copy.deepcopy(dict( + (k, v) for (k, v) in dict(ctx.node.properties.get(POLICY_FILTER, {})).iteritems() + if v or isinstance(v, (int, float)) + )) + if REQUEST_ID not in policy_filter: + policy_filter[REQUEST_ID] = str(uuid.uuid4()) + + policies_filtered = PolicyHandler.find_latest_policies(policy_filter) + + if not policies_filtered: + ctx.logger.info("policies not found by {0}".format(json.dumps(policy_filter))) + return True + + ctx.logger.info("found policies by {0}: {1}".format( + json.dumps(policy_filter), json.dumps(policies_filtered) + )) + ctx.instance.runtime_properties[POLICIES_FILTERED] = policies_filtered + + except Exception as ex: + error = "failed to find policies: {0}".format(str(ex)) + ctx.logger.error("{0}: {1}".format(error, traceback.format_exc())) + raise NonRecoverableError(error) + + return True + +######################################################### +@operation +def policy_get(**kwargs): + """retrieve the policy or policies and save it in runtime_properties""" + if ctx.type != NODE_INSTANCE: + error = "can only invoke policy_get on node of types: {0}".format(DCAE_POLICY_TYPES) + ctx.logger.error(error) + raise NonRecoverableError(error) + + if not _policy_get() and not _policies_find(): + error = "unexpected node type {0} for policy_get - expected types: {1}" \ + .format(ctx.node.type_hierarchy, DCAE_POLICY_TYPES) + ctx.logger.error(error) + raise NonRecoverableError(error) + + ctx.logger.info("exit policy_get") diff --git a/dcae-policy/pom.xml b/dcae-policy/pom.xml index 9573762..61a8c59 100644 --- a/dcae-policy/pom.xml +++ b/dcae-policy/pom.xml @@ -28,7 +28,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property. org.onap.dcaegen2.platform.plugins dcae-policy dcae-policy-plugin - 1.1.0-SNAPSHOT + 2.0.0-SNAPSHOT http://maven.apache.org UTF-8 diff --git a/dcae-policy/setup.py b/dcae-policy/setup.py index 528e744..9eea8bf 100644 --- a/dcae-policy/setup.py +++ b/dcae-policy/setup.py @@ -18,12 +18,14 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. +"""package for dcaepolicyplugin - getting policies from policy-engine through policy-handler""" + from setuptools import setup setup( name='dcaepolicyplugin', description='Cloudify plugin for dcae.nodes.policy node to retrieve the policy config', - version="1.0.0", + version="2.0.0", author='Alex Shatov', packages=['dcaepolicyplugin'], install_requires=[ diff --git a/dcae-policy/tests/__init__.py b/dcae-policy/tests/__init__.py new file mode 100644 index 0000000..a3220c4 --- /dev/null +++ b/dcae-policy/tests/__init__.py @@ -0,0 +1,18 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. diff --git a/dcae-policy/tests/log_ctx.py b/dcae-policy/tests/log_ctx.py index 9f5464d..9e8ef26 100644 --- a/dcae-policy/tests/log_ctx.py +++ b/dcae-policy/tests/log_ctx.py @@ -1,5 +1,3 @@ -""":@CtxLogger.log_ctx: decorator for logging the cloudify ctx before and after operation""" - # org.onap.dcae # ================================================================================ # Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. @@ -19,7 +17,10 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. +""":@CtxLogger.log_ctx: decorator for logging the cloudify ctx before and after operation""" + import json +import traceback from functools import wraps from cloudify import ctx @@ -102,7 +103,8 @@ class CtxLogger(object): ctx.logger.info("{0} context: {1}".format(\ func_name, json.dumps(CtxLogger.get_ctx_info()))) except Exception as ex: - ctx.logger.error("Failed to log the node context: {0}".format(str(ex))) + ctx.logger.error("Failed to log the node context: {0}: {1}" \ + .format(str(ex), traceback.format_exc())) @staticmethod def log_ctx(pre_log=True, after_log=False, exe_task=None): @@ -119,8 +121,8 @@ class CtxLogger(object): if ctx.type == NODE_INSTANCE and exe_task: ctx.instance.runtime_properties[exe_task] = func.__name__ except Exception as ex: - ctx.logger.error("Failed to set exe_task {0}: {1}".format(\ - exe_task, str(ex))) + ctx.logger.error("Failed to set exe_task {0}: {1}: {2}" \ + .format(exe_task, str(ex), traceback.format_exc())) if pre_log: CtxLogger.log_ctx_info('before ' + func.__name__) diff --git a/dcae-policy/tests/mock_cloudify_ctx.py b/dcae-policy/tests/mock_cloudify_ctx.py index 0c130c0..fe653d6 100644 --- a/dcae-policy/tests/mock_cloudify_ctx.py +++ b/dcae-policy/tests/mock_cloudify_ctx.py @@ -1,4 +1,3 @@ - # ============LICENSE_START======================================================= # org.onap.dcae # ================================================================================ @@ -19,6 +18,8 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. +"""mock cloudify context with relationships and type_hierarchy""" + from cloudify.mocks import MockCloudifyContext, MockNodeInstanceContext, MockNodeContext TARGET_NODE_ID = "target_node_id" diff --git a/dcae-policy/tests/test_tasks.py b/dcae-policy/tests/test_tasks.py index e2cc2e6..92d0611 100644 --- a/dcae-policy/tests/test_tasks.py +++ b/dcae-policy/tests/test_tasks.py @@ -18,28 +18,28 @@ # # ECOMP is a trademark and service mark of AT&T Intellectual Property. +"""unit tests for tasks in dcaepolicyplugin""" + import json import logging from datetime import datetime, timedelta import pytest - -from cloudify.state import current_ctx from cloudify.exceptions import NonRecoverableError - -from mock_cloudify_ctx import MockCloudifyContextFull, TARGET_NODE_ID, TARGET_NODE_NAME -from log_ctx import CtxLogger +from cloudify.state import current_ctx from dcaepolicyplugin import tasks +from tests.log_ctx import CtxLogger +from tests.mock_cloudify_ctx import (TARGET_NODE_ID, TARGET_NODE_NAME, + MockCloudifyContextFull) -DCAE_POLICY_TYPE = 'dcae.nodes.policy' POLICY_ID = 'policy_id' POLICY_VERSION = "policyVersion" POLICY_NAME = "policyName" POLICY_BODY = 'policy_body' POLICY_CONFIG = 'config' MONKEYED_POLICY_ID = 'monkeyed.Config_peach' -LOG_FILE = 'test_dcaepolicyplugin.log' +LOG_FILE = 'logs/test_dcaepolicyplugin.log' RUN_TS = datetime.utcnow() @@ -85,7 +85,7 @@ class MonkeyedPolicyBody(object): POLICY_VERSION: this_ver, POLICY_CONFIG: config, "matchingConditions": { - "ECOMPName": "DCAE", + "ONAPName": "DCAE", "ConfigName": "alex_config_name" }, "responseAttributes": {}, @@ -108,12 +108,15 @@ class MonkeyedPolicyBody(object): for key in policy_body_1.keys(): if key not in policy_body_2: return False - if isinstance(policy_body_1[key], dict): - return MonkeyedPolicyBody.is_the_same_dict( - policy_body_1[key], policy_body_2[key]) - if (policy_body_1[key] is None and policy_body_2[key] is not None) \ - or (policy_body_1[key] is not None and policy_body_2[key] is None) \ - or (policy_body_1[key] != policy_body_2[key]): + + val_1 = policy_body_1[key] + val_2 = policy_body_2[key] + if isinstance(val_1, dict) \ + and not MonkeyedPolicyBody.is_the_same_dict(val_1, val_2): + return False + if (val_1 is None and val_2 is not None) \ + or (val_1 is not None and val_2 is None) \ + or (val_1 != val_2): return False return True @@ -154,23 +157,34 @@ class MonkeyedNode(object): ) MonkeyedLogHandler.add_handler_to(self.ctx.logger) +def monkeyed_discovery_get_failure(full_path): + """monkeypatch for the GET to consul""" + return MonkeyedResponse(full_path, {}, None) + +def test_discovery_failure(monkeypatch): + """test finding policy-handler in consul""" + monkeypatch.setattr('requests.get', monkeyed_discovery_get_failure) + expected = None + tasks.PolicyHandler._lazy_init() + assert expected == tasks.PolicyHandler._url + def monkeyed_discovery_get(full_path): """monkeypatch for the GET to consul""" return MonkeyedResponse(full_path, {}, \ [{"ServiceAddress":"monkey-policy-handler-address", "ServicePort": "9999"}]) -def monkeyed_policy_handler_get(full_path, headers): - """monkeypatch for the GET to policy-engine""" - return MonkeyedResponse(full_path, headers, \ - MonkeyedPolicyBody.create_policy(MONKEYED_POLICY_ID)) - def test_discovery(monkeypatch): """test finding policy-handler in consul""" monkeypatch.setattr('requests.get', monkeyed_discovery_get) - expected = "http://monkey-policy-handler-address:9999/policy_latest" + expected = "http://monkey-policy-handler-address:9999" tasks.PolicyHandler._lazy_init() assert expected == tasks.PolicyHandler._url +def monkeyed_policy_handler_get(full_path, headers): + """monkeypatch for the GET to policy-engine""" + return MonkeyedResponse(full_path, headers, \ + MonkeyedPolicyBody.create_policy(MONKEYED_POLICY_ID)) + def test_policy_get(monkeypatch): """test policy_get operation on dcae.nodes.policy node""" monkeypatch.setattr('requests.get', monkeyed_policy_handler_get) @@ -178,7 +192,7 @@ def test_policy_get(monkeypatch): node_policy = MonkeyedNode( 'test_dcae_policy_node_id', 'test_dcae_policy_node_name', - DCAE_POLICY_TYPE, + tasks.DCAE_POLICY_TYPE, {POLICY_ID: MONKEYED_POLICY_ID} ) @@ -206,7 +220,55 @@ def test_policy_get(monkeypatch): with pytest.raises(NonRecoverableError) as excinfo: tasks.policy_get() CtxLogger.log_ctx_info("node_ms not policy type boom: {0}".format(str(excinfo.value))) - assert "can only invoke policy_get on node of type dcae.nodes.policy" in str(excinfo.value) + assert "unexpected node type " in str(excinfo.value) + + finally: + MockCloudifyContextFull.clear() + current_ctx.clear() + +def monkeyed_policy_handler_find(full_path, json, headers): + """monkeypatch for the GET to policy-engine""" + return MonkeyedResponse(full_path, headers, \ + {MONKEYED_POLICY_ID: MonkeyedPolicyBody.create_policy(MONKEYED_POLICY_ID)}) + +def test_policy_find(monkeypatch): + """test policy_get operation on dcae.nodes.policies node""" + monkeypatch.setattr('requests.post', monkeyed_policy_handler_find) + + node_policies = MonkeyedNode( + 'test_dcae_policies_node_id', + 'test_dcae_policies_node_name', + tasks.DCAE_POLICIES_TYPE, + {tasks.POLICY_FILTER: {POLICY_NAME: MONKEYED_POLICY_ID}} + ) + + try: + current_ctx.set(node_policies.ctx) + CtxLogger.log_ctx_info("before policy_get") + tasks.policy_get() + CtxLogger.log_ctx_info("after policy_get") + + expected = { + tasks.POLICIES_FILTERED: { + MONKEYED_POLICY_ID: MonkeyedPolicyBody.create_policy(MONKEYED_POLICY_ID)}} + + result = node_policies.ctx.instance.runtime_properties + node_policies.ctx.logger.info("expected runtime_properties: {0}".format( + json.dumps(expected))) + node_policies.ctx.logger.info("runtime_properties: {0}".format(json.dumps(result))) + assert MonkeyedPolicyBody.is_the_same_dict(result, expected) + assert MonkeyedPolicyBody.is_the_same_dict(expected, result) + + node_ms_multi = MonkeyedNode('test_ms_multi_id', 'test_ms_multi_name', "ms.nodes.type", \ + None, \ + [{TARGET_NODE_ID: node_policies.node_id, + TARGET_NODE_NAME: node_policies.node_name}]) + current_ctx.set(node_ms_multi.ctx) + CtxLogger.log_ctx_info("ctx of node_ms_multi not policy type") + with pytest.raises(NonRecoverableError) as excinfo: + tasks.policy_get() + CtxLogger.log_ctx_info("node_ms_multi not policy type boom: {0}".format(str(excinfo.value))) + assert "unexpected node type " in str(excinfo.value) finally: MockCloudifyContextFull.clear() diff --git a/dcae-policy/tox-local.ini b/dcae-policy/tox-local.ini index 70c1319..6bd1c58 100644 --- a/dcae-policy/tox-local.ini +++ b/dcae-policy/tox-local.ini @@ -1,3 +1,4 @@ +# tox -c tox-local.ini | tee -a logs/test_dcaepolicyplugin.log 2>&1 [tox] envlist = py27 @@ -12,5 +13,3 @@ setenv = PYTHONPATH={toxinidir} # recreate = True commands=pytest -v --cov dcaepolicyplugin --cov-report html - - -- cgit 1.2.3-korg