diff options
| -rw-r--r-- | cdap/.gitignore | 3 | ||||
| -rw-r--r-- | cdap/Changelog.md | 8 | ||||
| -rw-r--r-- | cdap/README.md | 2 | ||||
| -rwxr-xr-x | cdap/cdap_types.yaml | 2 | ||||
| -rw-r--r-- | cdap/cdapplugin/cdapcloudify/cdap_plugin.py | 74 | ||||
| -rw-r--r-- | cdap/cdapplugin/cdapcloudify/discovery.py | 45 | ||||
| -rw-r--r-- | cdap/cdapplugin/setup.py | 2 | ||||
| -rw-r--r-- | cdap/cdapplugin/tests/test_cdap_plugin.py (renamed from cdap/cdapplugin/tests/test_plugin.py) | 0 | ||||
| -rw-r--r-- | cdap/cdapplugin/tests/test_discovery.py | 115 | ||||
| -rw-r--r-- | cdap/cdapplugin/tox-local.ini | 10 | ||||
| -rw-r--r-- | cdap/cdapplugin/tox.ini | 8 | ||||
| -rw-r--r-- | cdap/demo_blueprints/cdap_hello_world.yaml | 4 | ||||
| -rw-r--r-- | cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml | 2 | ||||
| -rw-r--r-- | cdap/demo_blueprints/cdap_hello_world_with_laika.yaml | 4 |
14 files changed, 218 insertions, 61 deletions
diff --git a/cdap/.gitignore b/cdap/.gitignore index 0b11d9b..15622ff 100644 --- a/cdap/.gitignore +++ b/cdap/.gitignore @@ -1 +1,4 @@ cfyhelper.sh +cdapplugin/.coveragerc +cdapplugin/coverage-reports/ +cdapplugin/xunit-reports/* diff --git a/cdap/Changelog.md b/cdap/Changelog.md index b0006be..419cc5e 100644 --- a/cdap/Changelog.md +++ b/cdap/Changelog.md @@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [14.2.3] - Sep 14 2017 +* Remove the raise for status from discovery into tasks, allows for unit testing +* Unit test discovery + +## [14.2.2] - MISSING + +## [14.2.1] - MISSING + ## [14.2.0] * Integrate with Policy handler. Policy handling for CDAP is done. diff --git a/cdap/README.md b/cdap/README.md index 7aa357b..89481e3 100644 --- a/cdap/README.md +++ b/cdap/README.md @@ -175,4 +175,4 @@ The above (without having defined streams) will lead to: Note that the value is always a list of IP:Ports because there could be multiple identical services that satisfy the client (A in this case). This is client side load balancing. # Tests -To run the tests, you need `tox`. You can get it with `pip install tox`. After that, simply run `tox` from inside the `cdapplugin` directory to run the tests. +To run the tests, you need `tox`. You can get it with `pip install tox`. After that, simply run `tox -c tox-local.ini` from inside the `cdapplugin` directory to run the tests and generate a coverage report. diff --git a/cdap/cdap_types.yaml b/cdap/cdap_types.yaml index b2352be..aa01386 100755 --- a/cdap/cdap_types.yaml +++ b/cdap/cdap_types.yaml @@ -6,7 +6,7 @@ plugins: cdap_deploy: executor: central_deployment_agent package_name: cdapcloudify - package_version: 14.2.2 + package_version: 14.2.3 data_types: cdap_connections: diff --git a/cdap/cdapplugin/cdapcloudify/cdap_plugin.py b/cdap/cdapplugin/cdapcloudify/cdap_plugin.py index 6f4134e..00f80e2 100644 --- a/cdap/cdapplugin/cdapcloudify/cdap_plugin.py +++ b/cdap/cdapplugin/cdapcloudify/cdap_plugin.py @@ -27,6 +27,7 @@ import uuid import re from cdapcloudify import discovery import json +import requests # Property keys SERVICE_COMPONENT_NAME = "service_component_name" @@ -159,26 +160,31 @@ def deploy_and_start_application(**kwargs): #register with broker ctx.logger.info("Registering with Broker, config template was: {0}".format(json.dumps(config_template))) - discovery.put_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], - service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], - namespace = ctx.node.properties["namespace"], - streamname = ctx.node.properties["streamname"], - jar_url = ctx.node.properties["jar_url"], - artifact_name = ctx.node.properties["artifact_name"], - artifact_version = ctx.node.properties["artifact_version"], - app_config = config_template, - app_preferences = ctx.node.properties["app_preferences"], - service_endpoints = ctx.node.properties["service_endpoints"], - programs = ctx.node.properties["programs"], - program_preferences = ctx.node.properties["program_preferences"], - logger = ctx.logger) + response = discovery.put_broker( + cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], + service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], + namespace = ctx.node.properties["namespace"], + streamname = ctx.node.properties["streamname"], + jar_url = ctx.node.properties["jar_url"], + artifact_name = ctx.node.properties["artifact_name"], + artifact_version = ctx.node.properties["artifact_version"], + app_config = config_template, + app_preferences = ctx.node.properties["app_preferences"], + service_endpoints = ctx.node.properties["service_endpoints"], + programs = ctx.node.properties["programs"], + program_preferences = ctx.node.properties["program_preferences"], + logger = ctx.logger) + + response.raise_for_status() #bomb if not 2xx #TODO! Would be better to do an initial merge first before deploying, but the merge is complicated for CDAP #because of app config vs. app preferences. So, for now, let the broker do the work with an immediate reconfigure #get policies that may have changed prior to this blueprint deployment policy_configs = Policies.get_policy_configs() - ctx.logger.info("Updated policy configs: {0}".format(policy_configs)) - _trigger_update(policy_configs) + if policy_configs is not None: + ctx.logger.info("Updated policy configs: {0}".format(policy_configs)) + response = _trigger_update(policy_configs) + response.raise_for_status() #bomb if not 2xx except Exception as e: ctx.logger.error("Error depploying CDAP app: {er}".format(er=e)) @@ -189,11 +195,25 @@ def stop_and_undeploy_application(**kwargs): #per jack Lucas, do not raise Nonrecoverables on any delete operation. Keep going on them all, cleaning up as much as you can. #bombing would also bomb the deletion of the rest of the blueprint ctx.logger.info("Undeploying CDAP application") - try: #deregister with the broker, which will also take down the service from consul - discovery.delete_on_broker(ctx.instance.runtime_properties[SELECTED_BROKER], + response = discovery.delete_on_broker(ctx.instance.runtime_properties[SELECTED_BROKER], ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], ctx.logger) + response.raise_for_status() #bomb if not 2xx + except Exception as e: + ctx.logger.error("Error deregistering from Broker, but continuing with deletion process: {0}".format(e)) + +@operation +def delete_all_registered_apps(connected_broker_dns_name, **kwargs): + """ + Used in the cdap broker deleter node. + Deletes all registered applications (in the broker) + per jack Lucas, do not raise Nonrecoverables on any delete operation. Keep going on them all, cleaning up as much as you can. + """ + ctx.logger.info("Undeploying CDAP application") + try: + response = discovery.delete_all_registered_apps(connected_broker_dns_name, ctx.logger) + response.raise_for_status() #bomb if not 2xx except Exception as e: ctx.logger.error("Error deregistering from Broker, but continuing with deletion process: {0}".format(e)) @@ -211,11 +231,12 @@ def app_config_reconfigure(new_config_template, **kwargs): """ try: ctx.logger.info("Reconfiguring CDAP application via app_config") - discovery.reconfigure_in_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], + response = discovery.reconfigure_in_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], config = new_config_template, #This keyname will likely change per policy handler reconfiguration_type = "program-flowlet-app-config", logger = ctx.logger) + response.raise_for_status() #bomb if not 2xx except Exception as e: raise NonRecoverableError("CDAP Reconfigure error: {0}".format(e)) @@ -226,11 +247,12 @@ def app_preferences_reconfigure(new_config_template, **kwargs): """ try: ctx.logger.info("Reconfiguring CDAP application via app_preferences") - discovery.reconfigure_in_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], + response = discovery.reconfigure_in_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], config = new_config_template, #This keyname will likely change per policy handler reconfiguration_type = "program-flowlet-app-preferences", logger = ctx.logger) + response.raise_for_status() #bomb if not 2xx except Exception as e: raise NonRecoverableError("CDAP Reconfigure error: {0}".format(e)) @@ -241,7 +263,8 @@ def app_smart_reconfigure(new_config_template, **kwargs): """ try: ctx.logger.info("Reconfiguring CDAP application via smart interface") - _trigger_update([new_config_template]) + response = _trigger_update([new_config_template]) + response.raise_for_status() #bomb if not 2xx except Exception as e: raise NonRecoverableError("CDAP Reconfigure error: {0}".format(e)) @@ -253,15 +276,8 @@ def policy_update(updated_policies, **kwargs): try: #TODO! In the future, if we really have many different policies, would be more efficient to do a single merge here. #However all use cases today are a single policy so OK with this for loop for now. - _trigger_update(updated_policies) + response = _trigger_update(updated_policies) + response.raise_for_status() #bomb if not 2xx except Exception as e: raise NonRecoverableError("CDAP Reconfigure error: {0}".format(e)) -@operation -def delete_all_registered_apps(connected_broker_dns_name, **kwargs): - """ - Used in the cdap broker deleter node. - Deletes all registered applications (in the broker) - """ - discovery.delete_all_registered_apps(connected_broker_dns_name, ctx.logger) - diff --git a/cdap/cdapplugin/cdapcloudify/discovery.py b/cdap/cdapplugin/cdapcloudify/discovery.py index c654cbb..12daebc 100644 --- a/cdap/cdapplugin/cdapcloudify/discovery.py +++ b/cdap/cdapplugin/cdapcloudify/discovery.py @@ -5,9 +5,9 @@ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -46,18 +46,18 @@ def _get_broker_url(cdap_broker_name, service_component_name, logger): """ public """ -def put_broker(cdap_broker_name, - service_component_name, - namespace, - streamname, +def put_broker(cdap_broker_name, + service_component_name, + namespace, + streamname, jar_url, - artifact_name, + artifact_name, artifact_version, app_config, - app_preferences, - service_endpoints, - programs, - program_preferences, + app_preferences, + service_endpoints, + programs, + program_preferences, logger): """ Conforms to Broker API 4.X @@ -75,16 +75,17 @@ def put_broker(cdap_broker_name, data["services"] = service_endpoints data["programs"] = programs data["program_preferences"] = program_preferences - + #register with the broker response = requests.put(_get_broker_url(cdap_broker_name, service_component_name, logger), - json = data, + json = data, headers = {'content-type':'application/json'}) logger.info((response, response.status_code, response.text)) - response.raise_for_status() #bomb if not 2xx -def reconfigure_in_broker(cdap_broker_name, - service_component_name, + return response #let the caller deal with the response + +def reconfigure_in_broker(cdap_broker_name, + service_component_name, config, reconfiguration_type, logger): @@ -92,16 +93,17 @@ def reconfigure_in_broker(cdap_broker_name, #man am I glad I broke the broker API from 3 to 4 to standardize this interface because now I only need one function here response = requests.put("{u}/reconfigure".format(u = _get_broker_url(cdap_broker_name, service_component_name, logger)), headers = {'content-type':'application/json'}, - json = {"reconfiguration_type" : reconfiguration_type, + json = {"reconfiguration_type" : reconfiguration_type, "config" : config}) logger.info((response, response.status_code, response.text)) - response.raise_for_status() #bomb if not 2xx + + return response #let the caller deal with the response def delete_on_broker(cdap_broker_name, service_component_name, logger): #deregister with the broker response = requests.delete(_get_broker_url(cdap_broker_name, service_component_name, logger)) logger.info((response, response.status_code, response.text)) - response.raise_for_status() #bomb if not 2xx + return response def delete_all_registered_apps(cdap_broker_name, logger): #get the broker connection @@ -112,8 +114,9 @@ def delete_all_registered_apps(cdap_broker_name, logger): logger.info("Trying to connect to broker called {0} at {1}".format(cdap_broker_name, broker_url)) registered_apps = json.loads(requests.get("{0}/application".format(broker_url)).text) #should be proper list of strings (appnames) logger.info("Trying to delete: {0}".format(registered_apps)) - r = requests.post("{0}/application/delete".format(broker_url), + response = requests.post("{0}/application/delete".format(broker_url), headers = {'content-type':'application/json'}, json = {"appnames" : registered_apps}) - logger.info("Response: {0}, Response Status: {1}".format(r.text, r.status_code)) + logger.info("Response: {0}, Response Status: {1}".format(response.text, response.status_code)) + return response diff --git a/cdap/cdapplugin/setup.py b/cdap/cdapplugin/setup.py index 2121c7e..b683bd6 100644 --- a/cdap/cdapplugin/setup.py +++ b/cdap/cdapplugin/setup.py @@ -22,7 +22,7 @@ from setuptools import setup, find_packages setup( name = "cdapcloudify", - version = "14.2.2", + version = "14.2.3", packages=find_packages(), author = "Tommy Carpenter", author_email = "tommy at research dot eh tee tee dot com", diff --git a/cdap/cdapplugin/tests/test_plugin.py b/cdap/cdapplugin/tests/test_cdap_plugin.py index 7434fe8..7434fe8 100644 --- a/cdap/cdapplugin/tests/test_plugin.py +++ b/cdap/cdapplugin/tests/test_cdap_plugin.py diff --git a/cdap/cdapplugin/tests/test_discovery.py b/cdap/cdapplugin/tests/test_discovery.py new file mode 100644 index 0000000..7ee59c4 --- /dev/null +++ b/cdap/cdapplugin/tests/test_discovery.py @@ -0,0 +1,115 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from cdapcloudify import get_module_logger +from cdapcloudify import discovery +import pytest +import requests +import collections +import json + +logger = get_module_logger(__name__) + +_TEST_BROKER_NAME = "test_broker" +_TEST_SCN = "test_scn" + + +class FakeResponse: + def __init__(self, status_code, text, json = {}): + self.status_code = status_code + self.json = json #this is kind of misleading as the broker doesnt return the input as output but this cheat makes testing easier + self.text = text + +def _fake_putpost(url, json, headers): + return FakeResponse(status_code = 200, + json = json, + text = "URL: {0}, headers {1}".format(url, headers)) + +def _fake_delete(url): + return FakeResponse(status_code = 200, text = "URL: {0}".format(url)) + +def _fake_get_broker_url(cdap_broker_name, service_component_name, logger): + return "http://{ip}:{port}/application/{appname}".format(ip="666.666.666.666", port="666", appname=service_component_name) + +def test_put_broker(monkeypatch): + monkeypatch.setattr('requests.put', _fake_putpost) + monkeypatch.setattr('cdapcloudify.discovery._get_broker_url', _fake_get_broker_url) + R = discovery.put_broker( + _TEST_BROKER_NAME, + _TEST_SCN, + "test_ns", + "test_sn", + "test_ju", + "test_an", + "test_av", + "test_ac", + "test_ap", + "test_se", + "test_p", + "test_pp", + logger) + + assert R.text == "URL: http://666.666.666.666:666/application/test_scn, headers {'content-type': 'application/json'}" + assert R.json == {'app_preferences': 'test_ap', 'services': 'test_se', 'namespace': 'test_ns', 'programs': 'test_p', 'cdap_application_type': 'program-flowlet', 'app_config': 'test_ac', 'streamname': 'test_sn', 'program_preferences': 'test_pp', 'artifact_name': 'test_an', 'jar_url': 'test_ju', 'artifact_version': 'test_av'} + assert R.status_code == 200 + +def test_reconfigure_in_broker(monkeypatch): + monkeypatch.setattr('requests.put', _fake_putpost) + monkeypatch.setattr('cdapcloudify.discovery._get_broker_url', _fake_get_broker_url) + R = discovery.reconfigure_in_broker( + _TEST_BROKER_NAME, + _TEST_SCN, + {"redome" : "baby"}, + "program-flowlet-app-config", + logger) + assert R.text == "URL: http://666.666.666.666:666/application/test_scn/reconfigure, headers {'content-type': 'application/json'}" + assert R.json == {'reconfiguration_type': 'program-flowlet-app-config', 'config': {'redome': 'baby'}} + assert R.status_code == 200 + +def test_delete_on_broker(monkeypatch): + monkeypatch.setattr('requests.delete', _fake_delete) + monkeypatch.setattr('cdapcloudify.discovery._get_broker_url', _fake_get_broker_url) + R = discovery.delete_on_broker( + _TEST_BROKER_NAME, + _TEST_SCN, + logger) + print(R.text) + assert R.text == "URL: http://666.666.666.666:666/application/test_scn" + assert R.status_code == 200 + +def test_multi_delete(monkeypatch): + pretend_appnames = ['yo1', 'yo2'] + + def fake_get(url): + #return a fake list of app names + return FakeResponse(status_code = 200, + text = json.dumps(pretend_appnames)) + def fake_get_connection_info_from_consul(broker_name, logger): + return "666.666.666.666", "666" + + monkeypatch.setattr('requests.get', fake_get) + monkeypatch.setattr('cdapcloudify.discovery._get_connection_info_from_consul', fake_get_connection_info_from_consul) + monkeypatch.setattr('requests.post', _fake_putpost) + R = discovery.delete_all_registered_apps( + _TEST_BROKER_NAME, + logger) + + assert R.text == "URL: http://666.666.666.666:666/application/delete, headers {'content-type': 'application/json'}" + assert R.status_code == 200 + assert R.json == {'appnames': pretend_appnames} diff --git a/cdap/cdapplugin/tox-local.ini b/cdap/cdapplugin/tox-local.ini new file mode 100644 index 0000000..d553704 --- /dev/null +++ b/cdap/cdapplugin/tox-local.ini @@ -0,0 +1,10 @@ +[tox] +envlist = py27 +[testenv] +deps= + -rrequirements.txt + cloudify==3.4 + pytest + coverage + pytest-cov +commands=pytest --cov {envsitepackagesdir}/cdapcloudify --cov-report html diff --git a/cdap/cdapplugin/tox.ini b/cdap/cdapplugin/tox.ini index afabca4..246851e 100644 --- a/cdap/cdapplugin/tox.ini +++ b/cdap/cdapplugin/tox.ini @@ -2,7 +2,9 @@ envlist = py27 [testenv] deps= - pytest - uuid==1.30 + -rrequirements.txt cloudify==3.4 -commands=pytest + pytest + coverage + pytest-cov +commands=pytest --junitxml xunit-reports/xunit-result-cdapcloudify.xml --cov {envsitepackagesdir} --cov-report=xml diff --git a/cdap/demo_blueprints/cdap_hello_world.yaml b/cdap/demo_blueprints/cdap_hello_world.yaml index 4c78f38..5f75b08 100644 --- a/cdap/demo_blueprints/cdap_hello_world.yaml +++ b/cdap/demo_blueprints/cdap_hello_world.yaml @@ -2,8 +2,8 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml - - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.2.2/cdap_types.yaml - - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/dcaepolicy/0.0.1/node-type.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.2.3/cdap_types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/dcaepolicy/1.0.0/node-type.yaml inputs: hello_world_jar_url: diff --git a/cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml b/cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml index ea02543..b9e563e 100644 --- a/cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml +++ b/cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml @@ -2,7 +2,7 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml - - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.0.2/cdap_types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.2.3/cdap_types.yaml - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/dmaap/1.1.0/dmaap.yaml inputs: diff --git a/cdap/demo_blueprints/cdap_hello_world_with_laika.yaml b/cdap/demo_blueprints/cdap_hello_world_with_laika.yaml index 4587a47..f801a27 100644 --- a/cdap/demo_blueprints/cdap_hello_world_with_laika.yaml +++ b/cdap/demo_blueprints/cdap_hello_world_with_laika.yaml @@ -2,8 +2,8 @@ tosca_definitions_version: cloudify_dsl_1_3 imports: - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml - - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.0.2/cdap_types.yaml - - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/docker/2.1.0/node-type.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.2.3/cdap_types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/docker/2.3.0/node-type.yaml - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/relationship/1.0.0/node-type.yaml inputs: |