2.4.0 onap-dcae-dcaepolicy-lib

- discovering consul at consul:8500 instead of localhost:8500
- added and refactored unit tests for discovery -- coverage 94%

Change-Id: Id9f52cf95ab3dca0b2759021b408925a1dc87264
Signed-off-by: Alex Shatov <alexs@att.com>
Issue-ID: DCAEGEN2-438

4 files changed, 233 insertions, 36 deletions

diff --git a/onap-dcae-dcaepolicy-lib/onap_dcae_dcaepolicy_lib/policies_output.py b/onap-dcae-dcaepolicy-lib/onap_dcae_dcaepolicy_lib/policies_output.py
index dd82f5b..d8125c0 100644
--- a/onap-dcae-dcaepolicy-lib/onap_dcae_dcaepolicy_lib/policies_output.py
+++ b/onap-dcae-dcaepolicy-lib/onap_dcae_dcaepolicy_lib/policies_output.py
@@ -15,7 +15,8 @@
 # ============LICENSE_END=========================================================
 #
 # ECOMP is a trademark and service mark of AT&T Intellectual Property.
-"""client to talk to consul at the standard port 8500 on localhost"""
+
+"""client to talk to consul on standard port 8500"""
 
 import base64
 import json
@@ -29,59 +30,69 @@ from cloudify import ctx
 
 class PoliciesOutput(object):
     """static class for store-delete policies in consul kv"""
-    # it is safe to assume that consul agent is at localhost:8500 along with cloudify manager
-    CONSUL_TRANSACTION_URL = "http://localhost:8500/v1/txn"
+    # it is safe to assume that consul agent is at consul:8500
+    # define consul alis in /etc/hosts on cloudify manager vm
+    # $ cat /etc/hosts
+    # 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4 consul
+    CONSUL_TRANSACTION_URL = "http://consul:8500/v1/txn"
 
     POLICIES_EVENT = 'policies_event'
 
     POLICIES_FOLDER_MASK = "{0}:policies/{1}"
     MAX_OPS_PER_TXN = 64
-    # MAX_VALUE_LEN = 512 * 1000
 
     OPERATION_SET = "set"
     OPERATION_DELETE = "delete"
     OPERATION_DELETE_FOLDER = "delete-tree"
     SERVICE_COMPONENT_NAME = "service_component_name"
 
+
     @staticmethod
     def _gen_txn_operation(verb, service_component_name, key=None, value=None):
         """returns the properly formatted operation to be used inside transaction"""
-        key = PoliciesOutput.POLICIES_FOLDER_MASK.format(service_component_name, urllib.quote(key or ""))
+        key = PoliciesOutput.POLICIES_FOLDER_MASK.format(
+            service_component_name, urllib.quote(key or "")
+        )
         if value:
             return {"KV": {"Verb": verb, "Key": key, "Value": base64.b64encode(value)}}
         return {"KV": {"Verb": verb, "Key": key}}
 
+
     @staticmethod
     def _run_transaction(operation_name, txn):
         """run a single transaction of several operations at consul /txn"""
         if not txn:
-            return
+            return None
 
         response = None
         try:
             response = requests.put(PoliciesOutput.CONSUL_TRANSACTION_URL, json=txn)
         except requests.exceptions.RequestException as ex:
-            ctx.logger.error("failed to {0} at {1}: {2} on txn={3}"
+            ctx.logger.error(
+                "RequestException - failed to {0} at {1}: {2} on txn={3}"
                 .format(operation_name, PoliciesOutput.CONSUL_TRANSACTION_URL,
                         str(ex), json.dumps(txn)))
-            return
+            return None
 
         if response.status_code != requests.codes.ok:
-            ctx.logger.error("failed {0} {1}: {2} text={3} txn={4} headers={5}"
-                .format(operation_name, PoliciesOutput.CONSUL_TRANSACTION_URL, response.status_code,
-                        response.text, json.dumps(txn),
-                        json.dumps(dict(response.request.headers.items()))))
-            return
-        ctx.logger.info("response for {0} {1}: {2} text={3} txn={4} headers={5}"
-            .format(operation_name, PoliciesOutput.CONSUL_TRANSACTION_URL, response.status_code,
-                    response.text, json.dumps(txn),
-                    json.dumps(dict(response.request.headers.items()))))
+            ctx.logger.error(
+                "failed {0} for {1} {2}: text={3} txn={4}"
+                .format(response.status_code, operation_name,
+                        PoliciesOutput.CONSUL_TRANSACTION_URL, response.text, json.dumps(txn)))
+            return None
+        ctx.logger.info(
+            "response {0} for {1} {2}: text={3} txn={4}"
+            .format(response.status_code, operation_name,
+                    PoliciesOutput.CONSUL_TRANSACTION_URL, response.text, json.dumps(txn)))
         return True
 
+
     @staticmethod
     def store_policies(action, policy_bodies):
         """put the policy_bodies for service_component_name into consul-kv"""
-        service_component_name = ctx.instance.runtime_properties.get(PoliciesOutput.SERVICE_COMPONENT_NAME)
+        service_component_name = ctx.instance.runtime_properties.get(
+            PoliciesOutput.SERVICE_COMPONENT_NAME
+        )
         if not service_component_name:
             ctx.logger.warn("failed to find service_component_name to store_policies in consul-kv")
             return False
@@ -100,8 +111,10 @@ class PoliciesOutput(object):
             for policy_id, policy_body in policy_bodies.iteritems()
         ]
         txn = [
-            PoliciesOutput._gen_txn_operation(PoliciesOutput.OPERATION_DELETE_FOLDER, service_component_name),
-            PoliciesOutput._gen_txn_operation(PoliciesOutput.OPERATION_SET, service_component_name, "event", json.dumps(event))
+            PoliciesOutput._gen_txn_operation(
+                PoliciesOutput.OPERATION_DELETE_FOLDER, service_component_name),
+            PoliciesOutput._gen_txn_operation(
+                PoliciesOutput.OPERATION_SET, service_component_name, "event", json.dumps(event))
         ]
         idx_step = PoliciesOutput.MAX_OPS_PER_TXN - len(txn)
         for idx in xrange(0, len(store_policies), idx_step):
@@ -113,18 +126,23 @@ class PoliciesOutput(object):
         PoliciesOutput._run_transaction("store_policies", txn)
         return True
 
+
     @staticmethod
     def delete_policies():
         """delete policies for service_component_name in consul-kv"""
         if PoliciesOutput.POLICIES_EVENT not in ctx.instance.runtime_properties:
             return
 
-        service_component_name = ctx.instance.runtime_properties.get(PoliciesOutput.SERVICE_COMPONENT_NAME)
+        service_component_name = ctx.instance.runtime_properties.get(
+            PoliciesOutput.SERVICE_COMPONENT_NAME
+        )
         if not service_component_name:
             ctx.logger.warn("failed to find service_component_name to delete_policies in consul-kv")
             return
 
         delete_policies = [
-            PoliciesOutput._gen_txn_operation(PoliciesOutput.OPERATION_DELETE_FOLDER, service_component_name)
+            PoliciesOutput._gen_txn_operation(
+                PoliciesOutput.OPERATION_DELETE_FOLDER, service_component_name
+            )
         ]
         PoliciesOutput._run_transaction("delete_policies", delete_policies)
diff --git a/onap-dcae-dcaepolicy-lib/pom.xml b/onap-dcae-dcaepolicy-lib/pom.xml
index 8fce077..b7ae319 100644
--- a/onap-dcae-dcaepolicy-lib/pom.xml
+++ b/onap-dcae-dcaepolicy-lib/pom.xml
@@ -28,7 +28,7 @@ ECOMP is a trademark and service mark of AT&T Intellectual Property.
   <groupId>org.onap.dcaegen2.utils</groupId>
   <artifactId>onap-dcae-dcaepolicy-lib</artifactId>
   <name>dcaegen2-utils-onap-dcae-dcaepolicy-lib</name>
-  <version>2.3.0-SNAPSHOT</version>
+  <version>2.4.0-SNAPSHOT</version>
   <url>http://maven.apache.org</url>
 
   <properties>
diff --git a/onap-dcae-dcaepolicy-lib/setup.py b/onap-dcae-dcaepolicy-lib/setup.py
index 55dbaf3..a2ff93f 100644
--- a/onap-dcae-dcaepolicy-lib/setup.py
+++ b/onap-dcae-dcaepolicy-lib/setup.py
@@ -23,7 +23,7 @@ from setuptools import setup, find_packages
 setup(
     name='onap-dcae-dcaepolicy-lib',
     description='lib of policy decorators to be used by cloudify plugins of dcae controller',
-    version="2.3.0",
+    version="2.4.0",
     author='Alex Shatov',
     author_email="alexs@att.com",
     license='Apache 2',
diff --git a/onap-dcae-dcaepolicy-lib/tests/test_dcae_policy.py b/onap-dcae-dcaepolicy-lib/tests/test_dcae_policy.py
index 37ab9f6..4b3e39c 100644
--- a/onap-dcae-dcaepolicy-lib/tests/test_dcae_policy.py
+++ b/onap-dcae-dcaepolicy-lib/tests/test_dcae_policy.py
@@ -25,12 +25,14 @@ from datetime import datetime, timedelta
 from functools import wraps
 
 import pytest
+import requests
 from cloudify import ctx
 from cloudify.exceptions import NonRecoverableError
 from cloudify.state import current_ctx
 
 from onap_dcae_dcaepolicy_lib import dcae_policy
 from onap_dcae_dcaepolicy_lib.dcae_policy import Policies
+from onap_dcae_dcaepolicy_lib.policies_output import PoliciesOutput
 from tests.log_ctx import CtxLogger
 from tests.mock_cloudify_ctx import (TARGET_NODE_ID, TARGET_NODE_NAME,
                                      MockCloudifyContextFull)
@@ -167,6 +169,23 @@ class MonkeyedNode(object):
             runtime_properties=runtime_properties
         )
 
+class MonkeyedResponse(object):
+    """Monkey response"""
+    def __init__(self, full_path, headers=None, resp_json=None):
+        self.full_path = full_path
+        self.status_code = 200
+        self.headers = headers or {}
+        self.resp_json = resp_json
+        self.text = json.dumps(resp_json or {})
+
+    def json(self):
+        """returns json of response"""
+        return self.resp_json
+
+    def raise_for_status(self):
+        """always happy"""
+        pass
+
 def get_app_config():
     """just get the config"""
     config = copy.deepcopy(dict(ctx.instance.runtime_properties.get(APPLICATION_CONFIG, {})))
@@ -180,6 +199,7 @@ def operation_node_configure(**kwargs):
 
     app_config = get_app_config()
     ctx.instance.runtime_properties[APPLICATION_CONFIG] = app_config
+    ctx.instance.runtime_properties[PoliciesOutput.SERVICE_COMPONENT_NAME] = "unit_test_scn"
     ctx.logger.info("property app_config: {0}".format(json.dumps(app_config)))
 
 @CtxLogger.log_ctx(pre_log=True, after_log=True, exe_task='exe_task')
@@ -190,18 +210,6 @@ def node_configure(**kwargs):
     """
     operation_node_configure(**kwargs)
 
-@CtxLogger.log_ctx(pre_log=True, after_log=True, exe_task='exe_task')
-@Policies.gather_policies_to_node()
-def node_configure_wrong_order_path(**kwargs):
-    """wrong data in param policy_apply_order_path"""
-    operation_node_configure(**kwargs)
-
-@CtxLogger.log_ctx(pre_log=True, after_log=True, exe_task='exe_task')
-@Policies.gather_policies_to_node()
-def node_configure_empty_order_path(**kwargs):
-    """wrong data in param policy_apply_order_path"""
-    operation_node_configure(**kwargs)
-
 @CtxLogger.log_ctx(pre_log=True, after_log=True, exe_task='execute_operation')
 @Policies.update_policies_on_node()
 def policy_update(updated_policies, removed_policies=None, **kwargs):
@@ -242,6 +250,14 @@ def policy_update_many_calcs(updated_policies, removed_policies=None, policies=N
 
     ctx.instance.runtime_properties[APPLICATION_CONFIG] = app_config
 
+
+@CtxLogger.log_ctx(pre_log=True, after_log=True, exe_task='exe_task')
+@Policies.cleanup_policies_on_node
+def node_delete(**kwargs):
+    """delete <service_component_name> records in consul-kv"""
+    operation_node_configure(**kwargs)
+
+
 class CurrentCtx(object):
     """cloudify context"""
     _node_ms = None
@@ -445,6 +461,35 @@ class CurrentCtx(object):
         """reset context"""
         current_ctx.set(CurrentCtx._node_ms.ctx)
 
+
+def monkeyed_consul_boom(full_path, json):
+    """boom on monkeypatch for the put to consul"""
+    raise requests.ConnectionError("monkey-boom")
+
+
+@pytest.fixture()
+def fix_consul_boom(monkeypatch):
+    """monkeyed discovery request.put"""
+    PoliciesOutput._lazy_inited = False
+    monkeypatch.setattr('requests.put', monkeyed_consul_boom)
+    yield fix_consul_boom
+    PoliciesOutput._lazy_inited = False
+
+
+def monkeyed_consul_put(full_path, json):
+    """monkeypatch for the put to consul"""
+    return MonkeyedResponse(full_path)
+
+
+@pytest.fixture()
+def fix_consul(monkeypatch):
+    """monkeyed discovery request.put"""
+    PoliciesOutput._lazy_inited = False
+    monkeypatch.setattr('requests.put', monkeyed_consul_put)
+    yield fix_consul
+    PoliciesOutput._lazy_inited = False
+
+
 def cfy_ctx(include_bad=True, include_good=True):
     """test and safely clean up"""
     def cfy_ctx_decorator(func):
@@ -468,6 +513,7 @@ def cfy_ctx(include_bad=True, include_good=True):
         return ctx_wrapper
     return cfy_ctx_decorator
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_gather_policies_to_node():
     """test gather_policies_to_node"""
@@ -480,6 +526,7 @@ def test_gather_policies_to_node():
     policies = runtime_properties[dcae_policy.POLICIES]
     ctx.logger.info("policies: {0}".format(json.dumps(policies)))
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_policies_to_node():
     """test gather_policies_to_node"""
@@ -524,6 +571,7 @@ def test_policies_to_node():
     assert MonkeyedPolicyBody.is_the_same_dict(policy, expected_m)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_m, policy)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_update_policies():
     """test policy_update"""
@@ -584,6 +632,7 @@ def test_update_policies():
     assert MonkeyedPolicyBody.is_the_same_dict(policy, expected_b)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_b, policy)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_update_not_only_config():
     """test policy_update"""
@@ -644,6 +693,7 @@ def test_update_not_only_config():
     assert MonkeyedPolicyBody.is_the_same_dict(policy, expected_b)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_b, policy)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_update_policies_not():
     """test policy_update - ignore all policies with junk params"""
@@ -699,6 +749,7 @@ def test_update_policies_not():
     assert MonkeyedPolicyBody.is_the_same_dict(app_config, expected_app_config)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_app_config, app_config)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_update_many_calcs():
     """test policy_update"""
@@ -759,6 +810,7 @@ def test_update_many_calcs():
     assert MonkeyedPolicyBody.is_the_same_dict(policy, expected_b)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_b, policy)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_remove_all_policies():
     """test policy_update - remove all policies"""
@@ -788,6 +840,7 @@ def test_remove_all_policies():
     assert MonkeyedPolicyBody.is_the_same_dict(app_config, expected_config)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_config, app_config)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_remove_all_policies_twice():
     """test policy_update - remove all policies twice"""
@@ -818,6 +871,7 @@ def test_remove_all_policies_twice():
     assert MonkeyedPolicyBody.is_the_same_dict(app_config, expected_config)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_config, app_config)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_remove_then_update():
     """test policy_update"""
@@ -868,6 +922,7 @@ def test_remove_then_update():
     assert MONKEYED_POLICY_ID in policies
     assert MONKEYED_POLICY_ID_B in policies
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_remove_update_many_calcs():
     """test policy_update"""
@@ -924,6 +979,7 @@ def test_remove_update_many_calcs():
     assert MONKEYED_POLICY_ID in policies
     assert MONKEYED_POLICY_ID_B in policies
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True)
 def test_bad_update_many_calcs():
     """test policy_update"""
@@ -988,6 +1044,7 @@ def test_bad_update_many_calcs():
     assert MonkeyedPolicyBody.is_the_same_dict(policy, expected_b)
     assert MonkeyedPolicyBody.is_the_same_dict(expected_b, policy)
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True, include_good=False)
 def test_bad_policies():
     """test bad policy nodes"""
@@ -1000,6 +1057,7 @@ def test_bad_policies():
     policies = runtime_properties[dcae_policy.POLICIES]
     ctx.logger.info("policies: {0}".format(json.dumps(policies)))
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True, include_good=False)
 def test_wrong_ctx_node_configure():
     """test wrong ctx"""
@@ -1014,6 +1072,7 @@ def test_wrong_ctx_node_configure():
     assert ctx_type == 'cloudify.relationships.depends_on'
     assert str(excinfo.value) == "can only invoke gather_policies_to_node on node"
 
+@pytest.mark.usefixtures("fix_consul")
 @cfy_ctx(include_bad=True, include_good=False)
 def test_wrong_ctx_policy_update():
     """test wrong ctx"""
@@ -1105,3 +1164,123 @@ def test_defenses_on_set_policies():
     Policies._set_policies({})
 
     assert dcae_policy.POLICIES not in runtime_properties
+
+@pytest.mark.usefixtures("fix_consul")
+@cfy_ctx(include_bad=True)
+def test_delete_node():
+    """test delete"""
+    node_configure()
+
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES in runtime_properties
+    policies = runtime_properties[dcae_policy.POLICIES]
+    ctx.logger.info("policies: {0}".format(json.dumps(policies)))
+
+    node_delete()
+
+
+@pytest.mark.usefixtures("fix_consul_boom")
+@cfy_ctx(include_bad=True)
+def test_delete_node_no_consul():
+    """test delete without consul"""
+    node_configure()
+
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES in runtime_properties
+    policies = runtime_properties[dcae_policy.POLICIES]
+    ctx.logger.info("policies: {0}".format(json.dumps(policies)))
+
+    node_delete()
+
+
+@pytest.mark.usefixtures("fix_consul_boom")
+@cfy_ctx(include_bad=True)
+def test_delete_node_no_policies():
+    """test delete without consul and setup"""
+
+    ctx.instance.runtime_properties[PoliciesOutput.POLICIES_EVENT] = {}
+    ctx.instance.runtime_properties[PoliciesOutput.SERVICE_COMPONENT_NAME] = "delete_node_empty"
+
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES not in runtime_properties
+
+    node_delete()
+
+
+@pytest.mark.usefixtures("fix_consul_boom")
+@cfy_ctx(include_bad=True)
+def test_delete_node_empty():
+    """test delete without consul and setup"""
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES not in runtime_properties
+
+    node_delete()
+
+@pytest.mark.usefixtures("fix_consul_boom")
+@cfy_ctx(include_bad=True)
+def test_delete_node_lost_scn():
+    """test delete without consul and setup"""
+    ctx.instance.runtime_properties[PoliciesOutput.POLICIES_EVENT] = {}
+
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES not in runtime_properties
+
+    node_delete()
+
+
+@pytest.mark.usefixtures("fix_consul_boom")
+@cfy_ctx(include_bad=True)
+def test_delete_node_empty_config():
+    """test delete without consul and setup"""
+
+    ctx.instance.runtime_properties[PoliciesOutput.POLICIES_EVENT] = {}
+    ctx.instance.runtime_properties[PoliciesOutput.SERVICE_COMPONENT_NAME] = "delete_node_empty"
+
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES not in runtime_properties
+
+    node_delete()
+
+@pytest.mark.usefixtures("fix_consul_boom")
+@cfy_ctx(include_bad=True)
+def test_delete_ms_no_consul_addr():
+    """test delete without consul and setup"""
+
+    ctx.instance.runtime_properties[PoliciesOutput.POLICIES_EVENT] = {}
+    ctx.instance.runtime_properties[PoliciesOutput.SERVICE_COMPONENT_NAME] = "delete_node_empty"
+
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES not in runtime_properties
+
+    node_delete()
+
+
+@pytest.mark.usefixtures("fix_consul_boom")
+@cfy_ctx(include_bad=True)
+def test_delete_bad_config():
+    """test delete without consul and setup"""
+
+    ctx.instance.runtime_properties[PoliciesOutput.POLICIES_EVENT] = {}
+    ctx.instance.runtime_properties[PoliciesOutput.SERVICE_COMPONENT_NAME] = "delete_node_empty"
+
+    runtime_properties = ctx.instance.runtime_properties
+    ctx.logger.info("runtime_properties: {0}".format(json.dumps(runtime_properties)))
+
+    assert dcae_policy.POLICIES not in runtime_properties
+
+    node_delete()
+