diff options
| -rw-r--r-- | dcae-policy/.coveragerc | 29 | ||||
| -rw-r--r-- | dcae-policy/dcaepolicyplugin/tasks.py | 45 | ||||
| -rw-r--r-- | dcae-policy/tests/log_ctx.py | 134 | ||||
| -rw-r--r-- | dcae-policy/tests/mock_cloudify_ctx.py | 146 | ||||
| -rw-r--r-- | dcae-policy/tests/test_tasks.py | 213 | ||||
| -rw-r--r-- | dcae-policy/tox-local.ini | 16 | ||||
| -rw-r--r-- | dcae-policy/tox.ini | 15 |
7 files changed, 583 insertions, 15 deletions
diff --git a/dcae-policy/.coveragerc b/dcae-policy/.coveragerc new file mode 100644 index 0000000..2e600b5 --- /dev/null +++ b/dcae-policy/.coveragerc @@ -0,0 +1,29 @@ +# .coveragerc to control coverage.py +[run] +branch = True +cover_pylib = False +# include = */dcaepolicyplugin/*.py + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +ignore_errors = True + +[xml] +output = coverage-reports/coverage-dcaepolicyplugin.xml + diff --git a/dcae-policy/dcaepolicyplugin/tasks.py b/dcae-policy/dcaepolicyplugin/tasks.py index 7b5ea1f..2676864 100644 --- a/dcae-policy/dcaepolicyplugin/tasks.py +++ b/dcae-policy/dcaepolicyplugin/tasks.py @@ -32,28 +32,43 @@ from cloudify.exceptions import NonRecoverableError from .discovery import discover_service_url -SERVICE_NAME_POLICY_HANDLER = "policy_handler" -X_ECOMP_REQUESTID = 'X-ECOMP-RequestID' POLICY_ID = 'policy_id' POLICY_REQUIRED = 'policy_required' POLICY_BODY = 'policy_body' DCAE_POLICY_TYPE = 'dcae.nodes.policy' -POLICY_HANDLER_URL = discover_service_url(SERVICE_NAME_POLICY_HANDLER) +class PolicyHandler(object): + """talk to policy-handler""" + SERVICE_NAME_POLICY_HANDLER = "policy_handler" + X_ECOMP_REQUESTID = 'X-ECOMP-RequestID' + _url = None -def _get_latest_policy(policy_id): - """retrieve the latest policy for policy_id from policy-handler""" - ph_path = "{0}/policy_latest/{1}".format(POLICY_HANDLER_URL, policy_id) - headers = {X_ECOMP_REQUESTID: str(uuid.uuid4())} + @staticmethod + def _lazy_init(): + """discover policy-handler""" + if PolicyHandler._url: + return - ctx.logger.info("getting latest policy from {0} headers={1}".format( \ - ph_path, json.dumps(headers))) - res = requests.get(ph_path, headers=headers) - res.raise_for_status() + PolicyHandler._url = "{0}/policy_latest".format( + discover_service_url(PolicyHandler.SERVICE_NAME_POLICY_HANDLER) + ) - if res.status_code == requests.codes.ok: - return res.json() - return {} + @staticmethod + def get_latest_policy(policy_id): + """retrieve the latest policy for policy_id from policy-handler""" + PolicyHandler._lazy_init() + + ph_path = "{0}/{1}".format(PolicyHandler._url, policy_id) + headers = {PolicyHandler.X_ECOMP_REQUESTID: str(uuid.uuid4())} + + ctx.logger.info("getting latest policy from {0} headers={1}".format( \ + ph_path, json.dumps(headers))) + res = requests.get(ph_path, headers=headers) + res.raise_for_status() + + if res.status_code == requests.codes.ok: + return res.json() + return {} ######################################################### @operation @@ -71,7 +86,7 @@ def policy_get(**kwargs): try: policy_id = ctx.node.properties[POLICY_ID] - policy = _get_latest_policy(policy_id) + policy = PolicyHandler.get_latest_policy(policy_id) if not policy: raise NonRecoverableError("policy not found for policy_id {0}".format(policy_id)) diff --git a/dcae-policy/tests/log_ctx.py b/dcae-policy/tests/log_ctx.py new file mode 100644 index 0000000..9f5464d --- /dev/null +++ b/dcae-policy/tests/log_ctx.py @@ -0,0 +1,134 @@ +""":@CtxLogger.log_ctx: decorator for logging the cloudify ctx before and after operation""" + +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import json +from functools import wraps + +from cloudify import ctx +from cloudify.context import NODE_INSTANCE, RELATIONSHIP_INSTANCE + +class CtxLogger(object): + """static class for logging cloudify context ctx""" + @staticmethod + def _get_ctx_node_info(ctx_node): + if not ctx_node: + return {} + return {'id': ctx_node.id, 'name': ctx_node.name, 'type': ctx_node.type, + 'type_hierarchy': ctx_node.type_hierarchy, 'properties': ctx_node.properties} + + @staticmethod + def _get_ctx_instance_info(ctx_instance): + if not ctx_instance: + return {} + return {'id' : ctx_instance.id, 'runtime_properties' : ctx_instance.runtime_properties, + 'relationships' : CtxLogger._get_ctx_instance_relationships_info(ctx_instance)} + + @staticmethod + def _get_ctx_instance_relationships_info(ctx_instance): + if not ctx_instance or not ctx_instance.relationships: + return [] + return [{'target': CtxLogger._get_ctx_source_target_info(r.target), \ + 'type':r.type, 'type_hierarchy':r.type_hierarchy} \ + for r in ctx_instance.relationships] + + @staticmethod + def _get_ctx_source_target_info(ctx_source_target): + if not ctx_source_target: + return {} + return {'node': CtxLogger._get_ctx_node_info(ctx_source_target.node), + 'instance' : CtxLogger._get_ctx_instance_info(ctx_source_target.instance)} + + @staticmethod + def get_ctx_info(): + """collect the context data from ctx""" + context = { + 'type': ctx.type, + 'blueprint.id': ctx.blueprint.id, + 'deployment.id': ctx.deployment.id, + 'execution_id': ctx.execution_id, + 'workflow_id': ctx.workflow_id, + 'task_id': ctx.task_id, + 'task_name': ctx.task_name, + 'task_queue': ctx.task_queue, + 'task_target': ctx.task_target, + 'operation': { + 'name': ctx.operation.name, + 'retry_number': ctx.operation.retry_number, + 'max_retries': ctx.operation.max_retries + }, + 'plugin': { + 'name': ctx.plugin.name, + 'package_name': ctx.plugin.package_name, + 'package_version': ctx.plugin.package_version, + 'prefix': ctx.plugin.prefix, + 'workdir': ctx.plugin.workdir + } + } + if ctx.type == NODE_INSTANCE: + context['node'] = CtxLogger._get_ctx_node_info(ctx.node) + context['instance'] = CtxLogger._get_ctx_instance_info(ctx.instance) + elif ctx.type == RELATIONSHIP_INSTANCE: + context['source'] = CtxLogger._get_ctx_source_target_info(ctx.source) + context['target'] = CtxLogger._get_ctx_source_target_info(ctx.target) + + return context + + @staticmethod + def log_ctx_info(func_name): + """shortcut for logging of the ctx of the function""" + try: + if ctx.type == NODE_INSTANCE: + ctx.logger.info("{0} {1} context: {2}".format(\ + func_name, ctx.instance.id, json.dumps(CtxLogger.get_ctx_info()))) + elif ctx.type == RELATIONSHIP_INSTANCE: + ctx.logger.info("{0} context: {1}".format(\ + func_name, json.dumps(CtxLogger.get_ctx_info()))) + except Exception as ex: + ctx.logger.error("Failed to log the node context: {0}".format(str(ex))) + + @staticmethod + def log_ctx(pre_log=True, after_log=False, exe_task=None): + """Decorate each operation on the node to log the context - before and after. + Optionally save the current function name into runtime_properties[exe_task] + """ + def log_ctx_info_decorator(func, **arguments): + """Decorate each operation on the node to log the context""" + if func is not None: + @wraps(func) + def wrapper(*args, **kwargs): + """the actual logger before and after""" + try: + if ctx.type == NODE_INSTANCE and exe_task: + ctx.instance.runtime_properties[exe_task] = func.__name__ + except Exception as ex: + ctx.logger.error("Failed to set exe_task {0}: {1}".format(\ + exe_task, str(ex))) + if pre_log: + CtxLogger.log_ctx_info('before ' + func.__name__) + + result = func(*args, **kwargs) + + if after_log: + CtxLogger.log_ctx_info('after ' + func.__name__) + + return result + return wrapper + return log_ctx_info_decorator diff --git a/dcae-policy/tests/mock_cloudify_ctx.py b/dcae-policy/tests/mock_cloudify_ctx.py new file mode 100644 index 0000000..0c130c0 --- /dev/null +++ b/dcae-policy/tests/mock_cloudify_ctx.py @@ -0,0 +1,146 @@ + +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from cloudify.mocks import MockCloudifyContext, MockNodeInstanceContext, MockNodeContext + +TARGET_NODE_ID = "target_node_id" +TARGET_NODE_NAME = "target_node_name" + +class MockContextNode(MockNodeContext): + """ctx.node with type and type_hierarchy""" + + def __init__(self, id=None, properties=None, node_type=None, type_hierarchy=None): + super(MockContextNode, self).__init__(id, properties or {}) + self._type = node_type + self._type_hierarchy = type_hierarchy or [self._type] + MockCloudifyContextFull.nodes[id] = self + + @property + def type(self): + """node type""" + return self._type + + @property + def type_hierarchy(self): + """node type hierarchy is a list of types""" + return self._type_hierarchy + +class MockContextNodeInstance(MockNodeInstanceContext): + """ctx.instance with relationships""" + + def __init__(self, id=None, runtime_properties=None, relationships=None): + super(MockContextNodeInstance, self).__init__(id, runtime_properties or {}) + self._relationships = [] + self.add_relationships(relationships) + MockCloudifyContextFull.instances[id] = self + + def add_relationships(self, relationships): + """add more relationships to the node instance""" + if not relationships: + return + if not self._relationships: + self._relationships = [] + self._relationships.extend([ + MockContextRelationship(relationship) + for relationship in (relationships or []) if TARGET_NODE_ID in relationship + ]) + + @property + def relationships(self): + """list of relationships to other node instances""" + return self._relationships + +class MockContextRelationshipTarget(object): + """target of relationship""" + def __init__(self, relationship): + target_node_name = relationship[TARGET_NODE_NAME] + target_node_id = relationship[TARGET_NODE_ID] + + self.node = MockCloudifyContextFull.nodes.get(target_node_name) + self.instance = MockCloudifyContextFull.instances.get(target_node_id) + + if not self.node: + self.node = MockContextNode(target_node_name) + if not self.instance: + self.instance = MockContextNodeInstance(target_node_id) + +class MockContextRelationship(object): + """item of ctx.instance.relationships""" + + def __init__(self, relationship): + self.target = MockContextRelationshipTarget(relationship) + self.type = relationship.get("type", "cloudify.relationships.depends_on") + self.type_hierarchy = relationship.get("type_hierarchy") or [self.type] + +class MockCloudifyContextFull(MockCloudifyContext): + """ + ctx1 = MockCloudifyContextFull(node_id='node_1', + node_name='my_1', properties={'foo': 'bar'}) + ctx2 = MockCloudifyContextFull(node_id='node_2', + node_name='my_2', + relationships=[{'target_node_id': 'node_1', + 'target_node_name': 'my_1'}]) + """ + nodes = {} + instances = {} + + def __init__(self, + node_id=None, + node_name=None, + blueprint_id=None, + deployment_id=None, + execution_id=None, + properties=None, node_type=None, type_hierarchy=None, + runtime_properties=None, + capabilities=None, + related=None, + source=None, + target=None, + operation=None, + resources=None, + provider_context=None, + bootstrap_context=None, + relationships=None): + super(MockCloudifyContextFull, self).__init__( + node_id=node_id, + node_name=node_name, + blueprint_id=blueprint_id, + deployment_id=deployment_id, + execution_id=execution_id, + properties=properties, + capabilities=capabilities, + related=related, + source=source, + target=target, + operation=operation, + resources=resources, + provider_context=provider_context, + bootstrap_context=bootstrap_context, + runtime_properties=runtime_properties + ) + self._node = MockContextNode(node_name, properties, node_type, type_hierarchy) + self._instance = MockContextNodeInstance(node_id, runtime_properties, relationships) + + @staticmethod + def clear(): + """clean up the context links""" + MockCloudifyContextFull.instances.clear() + MockCloudifyContextFull.nodes.clear() diff --git a/dcae-policy/tests/test_tasks.py b/dcae-policy/tests/test_tasks.py new file mode 100644 index 0000000..e2cc2e6 --- /dev/null +++ b/dcae-policy/tests/test_tasks.py @@ -0,0 +1,213 @@ +# ============LICENSE_START======================================================= +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import json +import logging +from datetime import datetime, timedelta + +import pytest + +from cloudify.state import current_ctx +from cloudify.exceptions import NonRecoverableError + +from mock_cloudify_ctx import MockCloudifyContextFull, TARGET_NODE_ID, TARGET_NODE_NAME +from log_ctx import CtxLogger + +from dcaepolicyplugin import tasks + +DCAE_POLICY_TYPE = 'dcae.nodes.policy' +POLICY_ID = 'policy_id' +POLICY_VERSION = "policyVersion" +POLICY_NAME = "policyName" +POLICY_BODY = 'policy_body' +POLICY_CONFIG = 'config' +MONKEYED_POLICY_ID = 'monkeyed.Config_peach' +LOG_FILE = 'test_dcaepolicyplugin.log' + +RUN_TS = datetime.utcnow() + +class MonkeyedLogHandler(object): + """keep the shared logger handler here""" + _log_handler = None + + @staticmethod + def add_handler_to(logger): + """adds the local handler to the logger""" + if not MonkeyedLogHandler._log_handler: + MonkeyedLogHandler._log_handler = logging.FileHandler(LOG_FILE) + MonkeyedLogHandler._log_handler.setLevel(logging.DEBUG) + formatter = logging.Formatter( + fmt='%(asctime)s.%(msecs)03d %(levelname)+8s ' + \ + '%(threadName)s %(name)s.%(funcName)s: %(message)s', \ + datefmt='%Y%m%d_%H%M%S') + MonkeyedLogHandler._log_handler.setFormatter(formatter) + logger.addHandler(MonkeyedLogHandler._log_handler) + +class MonkeyedPolicyBody(object): + """policy body that policy-engine returns""" + @staticmethod + def create_policy_body(policy_id, policy_version=1): + """returns a fake policy-body""" + prev_ver = policy_version - 1 + timestamp = RUN_TS + timedelta(hours=prev_ver) + + prev_ver = str(prev_ver) + this_ver = str(policy_version) + config = { + "policy_updated_from_ver": prev_ver, + "policy_updated_to_ver": this_ver, + "policy_hello": "world!", + "policy_updated_ts": timestamp.isoformat()[:-3] + 'Z', + "updated_policy_id": policy_id + } + return { + "policyConfigMessage": "Config Retrieved! ", + "policyConfigStatus": "CONFIG_RETRIEVED", + "type": "JSON", + POLICY_NAME: "{0}.{1}.xml".format(policy_id, this_ver), + POLICY_VERSION: this_ver, + POLICY_CONFIG: config, + "matchingConditions": { + "ECOMPName": "DCAE", + "ConfigName": "alex_config_name" + }, + "responseAttributes": {}, + "property": None + } + + @staticmethod + def create_policy(policy_id, policy_version=1): + """returns the whole policy object for policy_id and policy_version""" + return { + POLICY_ID : policy_id, + POLICY_BODY : MonkeyedPolicyBody.create_policy_body(policy_id, policy_version) + } + + @staticmethod + def is_the_same_dict(policy_body_1, policy_body_2): + """check whether both policy_body objects are the same""" + if not isinstance(policy_body_1, dict) or not isinstance(policy_body_2, dict): + return False + for key in policy_body_1.keys(): + if key not in policy_body_2: + return False + if isinstance(policy_body_1[key], dict): + return MonkeyedPolicyBody.is_the_same_dict( + policy_body_1[key], policy_body_2[key]) + if (policy_body_1[key] is None and policy_body_2[key] is not None) \ + or (policy_body_1[key] is not None and policy_body_2[key] is None) \ + or (policy_body_1[key] != policy_body_2[key]): + return False + return True + +class MonkeyedResponse(object): + """Monkey response""" + def __init__(self, full_path, headers=None, resp_json=None): + self.full_path = full_path + self.status_code = 200 + self.headers = headers + self.resp_json = resp_json + + def json(self): + """returns json of response""" + return self.resp_json + + def raise_for_status(self): + """always happy""" + pass + +class MonkeyedNode(object): + """node in cloudify""" + BLUEPRINT_ID = 'test_dcae_policy_bp_id' + DEPLOYMENT_ID = 'test_dcae_policy_dpl_id' + EXECUTION_ID = 'test_dcae_policy_exe_id' + + def __init__(self, node_id, node_name, node_type, properties, relationships=None): + self.node_id = node_id + self.node_name = node_name + self.ctx = MockCloudifyContextFull( + node_id=self.node_id, + node_name=self.node_name, + node_type=node_type, + blueprint_id=MonkeyedNode.BLUEPRINT_ID, + deployment_id=MonkeyedNode.DEPLOYMENT_ID, + execution_id=MonkeyedNode.EXECUTION_ID, + properties=properties, + relationships=relationships + ) + MonkeyedLogHandler.add_handler_to(self.ctx.logger) + +def monkeyed_discovery_get(full_path): + """monkeypatch for the GET to consul""" + return MonkeyedResponse(full_path, {}, \ + [{"ServiceAddress":"monkey-policy-handler-address", "ServicePort": "9999"}]) + +def monkeyed_policy_handler_get(full_path, headers): + """monkeypatch for the GET to policy-engine""" + return MonkeyedResponse(full_path, headers, \ + MonkeyedPolicyBody.create_policy(MONKEYED_POLICY_ID)) + +def test_discovery(monkeypatch): + """test finding policy-handler in consul""" + monkeypatch.setattr('requests.get', monkeyed_discovery_get) + expected = "http://monkey-policy-handler-address:9999/policy_latest" + tasks.PolicyHandler._lazy_init() + assert expected == tasks.PolicyHandler._url + +def test_policy_get(monkeypatch): + """test policy_get operation on dcae.nodes.policy node""" + monkeypatch.setattr('requests.get', monkeyed_policy_handler_get) + + node_policy = MonkeyedNode( + 'test_dcae_policy_node_id', + 'test_dcae_policy_node_name', + DCAE_POLICY_TYPE, + {POLICY_ID: MONKEYED_POLICY_ID} + ) + + try: + current_ctx.set(node_policy.ctx) + CtxLogger.log_ctx_info("before policy_get") + tasks.policy_get() + CtxLogger.log_ctx_info("after policy_get") + + expected = { + POLICY_BODY : MonkeyedPolicyBody.create_policy_body(MONKEYED_POLICY_ID) + } + result = node_policy.ctx.instance.runtime_properties + node_policy.ctx.logger.info("expected runtime_properties: {0}".format( + json.dumps(expected))) + node_policy.ctx.logger.info("runtime_properties: {0}".format(json.dumps(result))) + assert MonkeyedPolicyBody.is_the_same_dict(result, expected) + assert MonkeyedPolicyBody.is_the_same_dict(expected, result) + + node_ms = MonkeyedNode('test_ms_id', 'test_ms_name', "ms.nodes.type", None, \ + [{TARGET_NODE_ID: node_policy.node_id, + TARGET_NODE_NAME: node_policy.node_name}]) + current_ctx.set(node_ms.ctx) + CtxLogger.log_ctx_info("ctx of node_ms not policy type") + with pytest.raises(NonRecoverableError) as excinfo: + tasks.policy_get() + CtxLogger.log_ctx_info("node_ms not policy type boom: {0}".format(str(excinfo.value))) + assert "can only invoke policy_get on node of type dcae.nodes.policy" in str(excinfo.value) + + finally: + MockCloudifyContextFull.clear() + current_ctx.clear() diff --git a/dcae-policy/tox-local.ini b/dcae-policy/tox-local.ini new file mode 100644 index 0000000..70c1319 --- /dev/null +++ b/dcae-policy/tox-local.ini @@ -0,0 +1,16 @@ +[tox] +envlist = py27 + +[testenv] +deps= + -rrequirements.txt + cloudify-plugins-common==3.4 + pytest + coverage + pytest-cov +setenv = + PYTHONPATH={toxinidir} +# recreate = True +commands=pytest -v --cov dcaepolicyplugin --cov-report html + + diff --git a/dcae-policy/tox.ini b/dcae-policy/tox.ini new file mode 100644 index 0000000..b78ba54 --- /dev/null +++ b/dcae-policy/tox.ini @@ -0,0 +1,15 @@ +# content of: tox.ini , put in same dir as setup.py +[tox] +envlist = py27 + +[testenv] +deps= + -rrequirements.txt + cloudify-plugins-common==3.4 + pytest + coverage + pytest-cov +setenv = + PYTHONPATH={toxinidir} +# recreate = True +commands=pytest --junitxml xunit-reports/xunit-result-dcaepolicyplugin.xml --cov dcaepolicyplugin --cov-report=xml |