From c272a87b84189f3bf8b3c7287199b816affc9926 Mon Sep 17 00:00:00 2001 From: Tommy Carpenter Date: Fri, 18 Aug 2017 14:16:44 -0400 Subject: [DCAEGEN2-74] Push up initial CDAP plugin Change-Id: I2bad410e5c55a59950cabe8fc13066954c4f5c92 Signed-off-by: Tommy Carpenter --- .gitignore | 67 +++++ cdap/.gitignore | 1 + cdap/Changelog.md | 37 +++ cdap/LICENSE.txt | 32 +++ cdap/README.md | 178 +++++++++++++ cdap/cdap_types.yaml | 90 +++++++ cdap/cdapplugin/cdapcloudify/__init__.py | 30 +++ cdap/cdapplugin/cdapcloudify/cdap_plugin.py | 231 +++++++++++++++++ cdap/cdapplugin/cdapcloudify/discovery.py | 105 ++++++++ cdap/cdapplugin/requirements.txt | 1 + cdap/cdapplugin/setup.py | 37 +++ cdap/cdapplugin/tests/test_plugin.py | 87 +++++++ cdap/cdapplugin/tox.ini | 8 + cdap/demo_blueprints/cdap_hello_world.yaml | 40 +++ .../cdap_hello_world_reconfigure.sh | 4 + .../cdap_hello_world_with_dmaap.yaml | 148 +++++++++++ .../cdap_hello_world_with_laika.yaml | 80 ++++++ cdap/demo_blueprints/cdap_hello_world_with_mr.yaml | 134 ++++++++++ cdap/pom.xml | 283 +++++++++++++++++++++ mvn-phase-script.sh | 152 +++++++++++ pom.xml | 24 ++ 21 files changed, 1769 insertions(+) create mode 100644 .gitignore create mode 100644 cdap/.gitignore create mode 100644 cdap/Changelog.md create mode 100644 cdap/LICENSE.txt create mode 100644 cdap/README.md create mode 100755 cdap/cdap_types.yaml create mode 100644 cdap/cdapplugin/cdapcloudify/__init__.py create mode 100644 cdap/cdapplugin/cdapcloudify/cdap_plugin.py create mode 100644 cdap/cdapplugin/cdapcloudify/discovery.py create mode 100644 cdap/cdapplugin/requirements.txt create mode 100644 cdap/cdapplugin/setup.py create mode 100644 cdap/cdapplugin/tests/test_plugin.py create mode 100644 cdap/cdapplugin/tox.ini create mode 100644 cdap/demo_blueprints/cdap_hello_world.yaml create mode 100755 cdap/demo_blueprints/cdap_hello_world_reconfigure.sh create mode 100644 cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml create mode 100644 cdap/demo_blueprints/cdap_hello_world_with_laika.yaml create mode 100644 cdap/demo_blueprints/cdap_hello_world_with_mr.yaml create mode 100644 cdap/pom.xml create mode 100755 mvn-phase-script.sh create mode 100644 pom.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d11997c --- /dev/null +++ b/.gitignore @@ -0,0 +1,67 @@ +.cloudify +*.swp +*.swn +*.swo +.DS_Store +.project +.pydevproject +venv + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ diff --git a/cdap/.gitignore b/cdap/.gitignore new file mode 100644 index 0000000..0b11d9b --- /dev/null +++ b/cdap/.gitignore @@ -0,0 +1 @@ +cfyhelper.sh diff --git a/cdap/Changelog.md b/cdap/Changelog.md new file mode 100644 index 0000000..d919964 --- /dev/null +++ b/cdap/Changelog.md @@ -0,0 +1,37 @@ +# Change Log +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/) +and this project adheres to [Semantic Versioning](http://semver.org/). + +## [14.0.2] +* Start a tox/pytest unit test suite + +## [14.0.1] +* Type file change to move reconfiguration defaults into the type file so each blueprint doesn't need them. + +## [14.0.0] +* Better type speccing in the type file +* Simplify the component naming +* Remove the unused (after two years) location and service-id properties +* Add more demo blueprints and reconfiguration tests + +## [13.0.0] +* Support for data router publication. Data router subscription is a problem, see README. +* Fixes `services_calls` to have the same format as streams. This is an API break but users are aware. + +## [12.1.0] +* Support for message router integration. Data router publish to come in next release. + +## [12.0.1] +* Use "localhost" instead of solutioning Consul host. + +## [12.0.0] +* Add in functions for policy to call (execute_workflows) to reconfigure CDAP applications +* Remove "Selected" Nonsense. + +FAILURE TO UPDATE + +## [10.0.0] +* Update to support broker API 3.X. This is a breaking change, involving the renaming of Node types +* Cut dependencies over to Nexus diff --git a/cdap/LICENSE.txt b/cdap/LICENSE.txt new file mode 100644 index 0000000..cb8008a --- /dev/null +++ b/cdap/LICENSE.txt @@ -0,0 +1,32 @@ +============LICENSE_START======================================================= +org.onap.dcae +================================================================================ +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +================================================================================ +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +============LICENSE_END========================================================= + +ECOMP is a trademark and service mark of AT&T Intellectual Property. + + +Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +=================================================================== +Licensed under the Creative Commons License, Attribution 4.0 Intl. (the "License"); +you may not use this documentation except in compliance with the License. +You may obtain a copy of the License at + https://creativecommons.org/licenses/by/4.0/ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/cdap/README.md b/cdap/README.md new file mode 100644 index 0000000..7aa357b --- /dev/null +++ b/cdap/README.md @@ -0,0 +1,178 @@ +# cdap-cloudify +Contains a plugin and type file for deploying CDAP and related artifacts. + +# service component name +When the cdap plugin deploys an application, it generates a service component name. That service component name is injected +into the node's runtime dictionary under the key "service_component_name" and also made available as an output under this key. + +# Demo blueprints +There is a subfolder in this repo called `demo_blueprints` that contains (templatized) example blueprints. + +# Connections +Since you cannot type-spec complicated objects in a cloudify node type, I have to explain this here. This is a requirement on all blueprints that use this node type. + +There is a property at the top level of the CDAP node called `connections` that is expecting a specific structure, best serviced with examples. + +## DMaaP + +### Message Router +Message router publication +``` + connections: + streams_publishes: // is a list + - name: topic00 // THIS NAME MUST MATCH THE NODE NAME IN BLUEPRINT, SEE BELOW* + location: mtc5 + client_role: XXXX + type: message_router + config_key: "myconfigkey1" // from spec + aaf_username: { get_input: aafu1 } + aaf_password: { get_input: aafp1 } + - name: topic01 // THIS NAME MUST MATCH THE NODE NAME IN BLUEPRINT, SEE BELOW* + location: mtc5 + client_role: XXXX + type: message_router + config_key: "myconfigkey2" // from spec + aaf_username: { get_input: aafu2 } + aaf_password: { get_input: aafp2 } +``` +Message router subscription is the exact same format, except change `streams_publishes` to `streams_subscribes`: +``` + streams_subscribes: + - name: topic00 #MEANT FOR DEMO ONLY! Subscribing and publishing to same topic. Not real example. + location: mtc5 + client_role: XXXX + type: message_router + config_key: "myconfigkey2" + aaf_username: { get_input: aafu2 } + aaf_password: { get_input: aafp2 } + - name: topic01 + location: mtc5 + client_role: XXXX + type: message_router + config_key: "myconfigkey3" + aaf_username: { get_input: aafu3 } + aaf_password: { get_input: aafp3 } +``` +The terms `streams_publishes` and `streams_subscribes` comes from the component specification. + +### Data Router +For publication, data router does not have the notion of AAF credentials, and there is no `client_role`. So the expected blueprint input is simpler than the MR case: +``` + streams_publishes: + ... + - name: feed00 + location: mtc5 + type: data_router + config_key: "mydrconfigkey" +``` + +Data router subscription is not supported because there is an impedance mistmatch between DR and CDAP. +CDAP streams expect a POST but DR outputs a PUT. +Some future platform capability needs to fill this hole; either something like the AF team's DR Sub or DMD. + +### Bound configuration +The above blueprint snippets will lead to the cdap application's `app_config` getting an entry that looks like this: +``` +{ + "streams_subscribes":{ + "myconfigkey3":{ + "type":"message_router", + "aaf_username":"foo3", + "aaf_password":"bar3", + "dmaap_info":{ + "client_role":"XXXX", + "client_id":"XXXX", + "location":"XXXX", + "topic_url":"XXXX" + } + }, + "myconfigkey2":{ + "type":"message_router", + "aaf_username":"foo2", + "aaf_password":"bar2", + "dmaap_info":{ + "client_role":"XXXX", + "client_id":"XXXX", + "location":"XXXX", + "topic_url":"XXXX" + } + } + }, + "streams_publishes":{ + "myconfigkey1":{ + "type":"message_router", + "aaf_username":"foo1", + "aaf_password":"bar1", + "dmaap_info":{ + "client_role":"XXXX", + "client_id":"XXXX", + "location":"XXXX", + "topic_url":"XXXX" + } + }, + "mydrconfigkey":{ + "type":"data_router", + "dmaap_info":{ + "username":"XXXX", + "location":"XXXX", + "publish_url":"XXXX", + "publisher_id":"XXXX", + "log_url":"XXXX", + "password":"XXXX" + } + }, + "myconfigkey0":{ + "type":"message_router", + "aaf_username":"foo0", + "aaf_password":"bar0", + "dmaap_info":{ + "client_role":"XXXX", + "client_id":"XXXX", + "location":"XXXX", + "topic_url":"XXXX" + } + } + } +} +``` +## HTTP +In addition to DMaaP, we support HTTP services. + +### Services Calls +In a blueprint, to express that one component calls asynchronous HTTP service of another component, writing this as `A -> B,` you need: + +1. `A` to have a `connections/services_calls` entry: +``` + connections: + services_calls: + - service_component_type: laika + config_key: "laika_handle" +``` +2. A relationship of type `dcae.relationships.component_connected_to` from A to B. + +3. The `B` node's `service_component_type` should match #1 + +See the demo blueprint `cdap_hello_world_with_laika.yaml` + +### Bound Configuration + +The above (without having defined streams) will lead to: +``` +{ + "streams_subscribes":{ + + }, + "streams_publishes":{ + + }, + "services_calls":{ + "laika_handle":[ + "some_up:some_port" + ] + } +} +``` +Note that the value is always a list of IP:Ports because there could be multiple identical services that satisfy the client (A in this case). This is client side load balancing. + +# Tests +To run the tests, you need `tox`. You can get it with `pip install tox`. After that, simply run `tox` from inside the `cdapplugin` directory to run the tests. diff --git a/cdap/cdap_types.yaml b/cdap/cdap_types.yaml new file mode 100755 index 0000000..497307c --- /dev/null +++ b/cdap/cdap_types.yaml @@ -0,0 +1,90 @@ +tosca_definitions_version: cloudify_dsl_1_3 + +imports: + - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml +plugins: + cdap_deploy: + executor: central_deployment_agent + package_name: cdapcloudify + package_version: 14.0.2 + +data_types: + cdap_connections: + properties: + services_calls: + default: [] + streams_publishes: + default: [] + streams_subscribes: + default: [] + +node_types: + dcae.nodes.MicroService.cdap: + derived_from: cloudify.nodes.Root + properties: + service_component_type: + type: string + ##### + #For the following parameters in this block, see the Broker API + ##### + jar_url: + type: string + artifact_name: + type: string + artifact_version: + type: string + connections: + type: cdap_connections + app_config: + default: {} + app_preferences: + default: {} + program_preferences: + default: [] + programs: + default: [] + streamname: + #currently, we only support CDAP apps written that read from a + #stream. This is not the only ingest mechanism for CDAP. This may have to change/get + type: string + namespace: + #the namespace to deploy the CDAP app into + #defaults to the default cdap namespace which is called "default" + type: string + default : "default" + service_endpoints: + default: [] + + interfaces: + cloudify.interfaces.lifecycle: + create: + implementation: cdap_deploy.cdapcloudify.cdap_plugin.create + inputs: + connected_broker_dns_name: + type: string + description: This is the broker's DNS name. There could be multiple brokers/clusters at a site. Could by populated via an intrinsic_function in a blueprint, or manually via inputs file + default: "cdap_broker" + start: + cdap_deploy.cdapcloudify.cdap_plugin.deploy_and_start_application + delete: + cdap_deploy.cdapcloudify.cdap_plugin.stop_and_undeploy_application + reconfiguration: + app_config_reconfigure: + implementation: cdap_deploy.cdapcloudify.cdap_plugin.app_config_reconfigure + inputs: + new_config_template: + description: "new unbound config for the CDAP AppConfig as a JSON" + default: {} + app_preferences_reconfigure: + implementation: cdap_deploy.cdapcloudify.cdap_plugin.app_preferences_reconfigure + inputs: + new_config_template: + description: "new bound config for the CDAP AppPreferences as a JSON" + default: {} + app_smart_reconfigure: + implementation: cdap_deploy.cdapcloudify.cdap_plugin.app_smart_reconfigure + inputs: + new_config_template: + description: "new unbound config for the CDAP AppConfig as a JSON" + default: {} + diff --git a/cdap/cdapplugin/cdapcloudify/__init__.py b/cdap/cdapplugin/cdapcloudify/__init__.py new file mode 100644 index 0000000..388ac55 --- /dev/null +++ b/cdap/cdapplugin/cdapcloudify/__init__.py @@ -0,0 +1,30 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import logging + +def get_module_logger(mod_name): + logger = logging.getLogger(mod_name) + handler = logging.StreamHandler() + formatter = logging.Formatter( + '%(asctime)s [%(name)-12s] %(levelname)-8s %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + return logger diff --git a/cdap/cdapplugin/cdapcloudify/cdap_plugin.py b/cdap/cdapplugin/cdapcloudify/cdap_plugin.py new file mode 100644 index 0000000..f5eaf0b --- /dev/null +++ b/cdap/cdapplugin/cdapcloudify/cdap_plugin.py @@ -0,0 +1,231 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import requests +from cloudify import ctx +from cloudify.decorators import operation +from cloudify.exceptions import NonRecoverableError +import time +import uuid +import re +from cdapcloudify import discovery +import json + +""" +TODO: Tons of crappy URL forming going on here... +""" +# Property keys +SERVICE_COMPONENT_NAME = "service_component_name" +SELECTED_BROKER = "selected_broker" +PUB_C = "streams_publishes_for_config" +SUB_C = "streams_subscribes_for_config" +SER_C = "services_calls_for_config" +STREAMS_PUBLISHES = "streams_publishes" +STREAMS_SUBSCRIBES = "streams_subscribes" +SERVICES_CALLS = "services_calls" + +# Custom Exception +class BadConnections(NonRecoverableError): + pass + + +def _validate_conns(connections): + """ + Cloudify allows you to type spec a data type in a type file, however it does not appear to do strict checking on blueprints against that. + Sad! + The "connections" block has an important structure to this plugin, so here we validate it and fail fast if it is not correct. + """ + try: + def _assert_ks_in_d(ks,d): + for k in ks: + assert(k in d) + assert STREAMS_PUBLISHES in connections + assert STREAMS_SUBSCRIBES in connections + for s in connections[STREAMS_PUBLISHES] + connections[STREAMS_SUBSCRIBES]: + _assert_ks_in_d(["name", "location", "type", "config_key"], s) + assert(s["type"] in ["message_router", "data_router"]) + if s["type"] == "message_router": + _assert_ks_in_d(["aaf_username", "aaf_password", "client_role"], s) #I am not checking that these are not blank. I will leave it possible for you to put empty values for these, but force you to acknowledge that you are doing so by not allowing these to be ommited. + #nothing extra for DR; no AAF, no client role. + except: + raise BadConnections("Bad Connections definition in blueprint") #is a NoneRecoverable + +def _streams_iterator(streams): + """ + helper function for iterating over streams_publishes and subscribes + note! this is an impure function. it also sets the properties the dmaap plugin needs into runtime properties + """ + for_config = {} + for s in streams: + if s["type"] == "message_router": + #set the properties the DMaaP plugin needs + ctx.instance.runtime_properties[s["name"]] = {"client_role" : s["client_role"], "location" : s["location"]} + #form (or append to) the dict the component will get, including the template for the CBS + for_config[s["config_key"]] = {"aaf_username" : s["aaf_username"], "aaf_password" : s["aaf_password"], "type" : s["type"], "dmaap_info" : "<< " + s["name"] + ">>"} #will get bound by CBS + if s["type"] == "data_router": + #set the properties the DMaaP plugin needs$ + ctx.instance.runtime_properties[s["name"]] = {"location" : s["location"]} + #form (or append to) the dict the component will get, including the template for the CBS$ + for_config[s["config_key"]] = {"type" : s["type"], "dmaap_info" : "<<" + s["name"] + ">>"} #will get bound by CBS + + return for_config + +def _services_calls_iterator(services_calls): + """ + helper function for iterating over services_calls + """ + for_config = {} + for s in services_calls: + #form (or append to) the dict the component will get, including the template for the CBS + for_config[s["config_key"]] = "{{ " + s["service_component_type"] + " }}" #will get bound by CBS + return for_config + +###################### +# Cloudify Operations +###################### + +@operation +def create(connected_broker_dns_name, **kwargs): + """ + This is apparantly needed due to the order in which Cloudify relationships are handled in Cloudify. + """ + + #fail fast + _validate_conns(ctx.node.properties["connections"]) + + #The config binding service needs to know whether cdap or docker. Currently (aug 1 2018) it looks for "cdap_app" in the name + service_component_name = "{0}_cdap_app_{1}".format(str(uuid.uuid4()).replace("-",""), ctx.node.properties["service_component_type"]) + + #set this into a runtime dictionary + ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME] = service_component_name + + #fetch the broker name from inputs and set it in runtime properties so other functions can use it + ctx.instance.runtime_properties[SELECTED_BROKER] = connected_broker_dns_name + + #set the properties the DMaap plugin expects for message router + #see the README for the structures of these keys + #NOTE! This has to be done in create because Jack's DMaaP plugin expects to do it's thing in preconfigure. + # and we need to get this key into consul before start + #set this as a runtime property for start to use + ctx.instance.runtime_properties[PUB_C] = _streams_iterator(ctx.node.properties["connections"][STREAMS_PUBLISHES]) + ctx.instance.runtime_properties[SUB_C] = _streams_iterator(ctx.node.properties["connections"][STREAMS_SUBSCRIBES]) + ctx.instance.runtime_properties[SER_C] = _services_calls_iterator(ctx.node.properties["connections"][SERVICES_CALLS]) + +@operation +def deploy_and_start_application(**kwargs): + """ + pushes the application into the workspace and starts it + """ + try: + #parse TOSCA model params + config_template = ctx.node.properties["app_config"] + + #there is a typed section in the node type called "connections", but the broker expects those two keys at the top level of app_config, so add them here + #In cloudify you can't have a custom data type and then specify unknown propertys, the vlidation will fail, so typespeccing just part of app_config doesnt work + #the rest of the CDAP app's app_config is app-dependent + config_template[SERVICES_CALLS] = ctx.instance.runtime_properties[SER_C] + config_template[STREAMS_PUBLISHES] = ctx.instance.runtime_properties[PUB_C] + config_template[STREAMS_SUBSCRIBES] = ctx.instance.runtime_properties[SUB_C] + + #register with broker + ctx.logger.info("Registering with Broker, config template was: {0}".format(json.dumps(config_template))) + discovery.put_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], + service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], + namespace = ctx.node.properties["namespace"], + streamname = ctx.node.properties["streamname"], + jar_url = ctx.node.properties["jar_url"], + artifact_name = ctx.node.properties["artifact_name"], + artifact_version = ctx.node.properties["artifact_version"], + app_config = config_template, + app_preferences = ctx.node.properties["app_preferences"], + service_endpoints = ctx.node.properties["service_endpoints"], + programs = ctx.node.properties["programs"], + program_preferences = ctx.node.properties["program_preferences"], + logger = ctx.logger) + + except Exception as e: + ctx.logger.error("Error depploying CDAP app: {er}".format(er=e)) + raise NonRecoverableError(e) + +@operation +def stop_and_undeploy_application(**kwargs): + #per jack Lucas, do not raise Nonrecoverables on any delete operation. Keep going on them all, cleaning up as much as you can. + #bombing would also bomb the deletion of the rest of the blueprint + ctx.logger.info("Undeploying CDAP application") + + try: #deregister with the broker, which will also take down the service from consul + discovery.delete_on_broker(ctx.instance.runtime_properties[SELECTED_BROKER], + ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], + ctx.logger) + except Exception as e: + ctx.logger.error("Error deregistering from Broker, but continuing with deletion process: {0}".format(e)) + +############ +#RECONFIGURATION +# These calls works as follows: +# 1) it expects "new_config_template" to be a key in kwargs, i.e., passed in using execute_operations -p parameter +# 2) it pushes the new unbound config down to the broker +# 3) broker deals with the rest +############ +@operation +def app_config_reconfigure(new_config_template, **kwargs): + """ + reconfigure the CDAP app's app config + """ + try: + ctx.logger.info("Reconfiguring CDAP application via app_config") + discovery.reconfigure_in_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], + service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], + config = new_config_template, #This keyname will likely change per policy handler + reconfiguration_type = "program-flowlet-app-config", + logger = ctx.logger) + except Exception as e: + raise NonRecoverableError("CDAP Reconfigure error: {0}".format(e)) + +@operation +def app_preferences_reconfigure(new_config_template, **kwargs): + """ + reconfigure the CDAP app's app preferences + """ + try: + ctx.logger.info("Reconfiguring CDAP application via app_preferences") + discovery.reconfigure_in_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], + service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], + config = new_config_template, #This keyname will likely change per policy handler + reconfiguration_type = "program-flowlet-app-preferences", + logger = ctx.logger) + except Exception as e: + raise NonRecoverableError("CDAP Reconfigure error: {0}".format(e)) + +@operation +def app_smart_reconfigure(new_config_template, **kwargs): + """ + reconfigure the CDAP app via the broker smart interface + """ + try: + ctx.logger.info("Reconfiguring CDAP application via smart interface") + discovery.reconfigure_in_broker(cdap_broker_name = ctx.instance.runtime_properties[SELECTED_BROKER], + service_component_name = ctx.instance.runtime_properties[SERVICE_COMPONENT_NAME], + config = new_config_template, #This keyname will likely change per policy handler + reconfiguration_type = "program-flowlet-smart", + logger = ctx.logger) + except Exception as e: + raise NonRecoverableError("CDAP Reconfigure error: {0}".format(e)) + + diff --git a/cdap/cdapplugin/cdapcloudify/discovery.py b/cdap/cdapplugin/cdapcloudify/discovery.py new file mode 100644 index 0000000..a8f0ce2 --- /dev/null +++ b/cdap/cdapplugin/cdapcloudify/discovery.py @@ -0,0 +1,105 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import requests +import json + +CONSUL_HOST = "http://localhost:8500" + +def _get_broker_url(cdap_broker_name, service_component_name, logger): + """ + fetch the broker connection information from Consul + """ + def _get_connection_info_from_consul(service_component_name, logger): + """ + Call consul's catalog + TODO: currently assumes there is only one service + """ + url = "{0}/v1/catalog/service/{1}".format(CONSUL_HOST, service_component_name) + logger.info("Trying to query: {0}".format(url)) + res = requests.get(url) + res.raise_for_status() + services = res.json() + return services[0]["ServiceAddress"], services[0]["ServicePort"] + + broker_ip, broker_port = _get_connection_info_from_consul(cdap_broker_name, logger) + broker_url = "http://{ip}:{port}/application/{appname}".format(ip=broker_ip, port=broker_port, appname=service_component_name) + logger.info("Trying to connect to broker endpoint: {0}".format(broker_url)) + return broker_url + +""" +public +""" +def put_broker(cdap_broker_name, + service_component_name, + namespace, + streamname, + jar_url, + artifact_name, + artifact_version, + app_config, + app_preferences, + service_endpoints, + programs, + program_preferences, + logger): + """ + Conforms to Broker API 4.X + """ + + data = dict() + data["cdap_application_type"] = "program-flowlet" + data["namespace"] = namespace + data["streamname"] = streamname + data["jar_url"] = jar_url + data["artifact_name"] = artifact_name + data["artifact_version"] = artifact_version + data["app_config"] = app_config + data["app_preferences"] = app_preferences + data["services"] = service_endpoints + data["programs"] = programs + data["program_preferences"] = program_preferences + + #register with the broker + response = requests.put(_get_broker_url(cdap_broker_name, service_component_name, logger), + json = data, + headers = {'content-type':'application/json'}) + logger.info((response, response.status_code, response.text)) + response.raise_for_status() #bomb if not 2xx + +def reconfigure_in_broker(cdap_broker_name, + service_component_name, + config, + reconfiguration_type, + logger): + #trigger a reconfiguration with the broker + #man am I glad I broke the broker API from 3 to 4 to standardize this interface because now I only need one function here + response = requests.put("{u}/reconfigure".format(u = _get_broker_url(cdap_broker_name, service_component_name, logger)), + headers = {'content-type':'application/json'}, + json = {"reconfiguration_type" : reconfiguration_type, + "config" : config}) + logger.info((response, response.status_code, response.text)) + response.raise_for_status() #bomb if not 2xx + +def delete_on_broker(cdap_broker_name, service_component_name, logger): + #deregister with the broker + response = requests.delete(_get_broker_url(cdap_broker_name, service_component_name, logger)) + logger.info((response, response.status_code, response.text)) + response.raise_for_status() #bomb if not 2xx + diff --git a/cdap/cdapplugin/requirements.txt b/cdap/cdapplugin/requirements.txt new file mode 100644 index 0000000..1128300 --- /dev/null +++ b/cdap/cdapplugin/requirements.txt @@ -0,0 +1 @@ +uuid==1.30 diff --git a/cdap/cdapplugin/setup.py b/cdap/cdapplugin/setup.py new file mode 100644 index 0000000..d16667d --- /dev/null +++ b/cdap/cdapplugin/setup.py @@ -0,0 +1,37 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +import os +from setuptools import setup, find_packages + +setup( + name = "cdapcloudify", + version = "14.0.2", + packages=find_packages(), + author = "Tommy Carpenter", + author_email = "tommy at research dot eh tee tee dot com", + description = ("Cloudify plugin for CDAP"), + license = "", + keywords = "", + url = "https://gerrit.onap.org/r/#/admin/projects/dcaegen2/platform/plugins", + zip_safe=False, + install_requires = [ + "uuid==1.30" + ] +) diff --git a/cdap/cdapplugin/tests/test_plugin.py b/cdap/cdapplugin/tests/test_plugin.py new file mode 100644 index 0000000..7434fe8 --- /dev/null +++ b/cdap/cdapplugin/tests/test_plugin.py @@ -0,0 +1,87 @@ +# org.onap.dcae +# ================================================================================ +# Copyright (c) 2017 AT&T Intellectual Property. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +# +# ECOMP is a trademark and service mark of AT&T Intellectual Property. + +from cdapcloudify.cdap_plugin import _validate_conns, BadConnections +import pytest + +#todo.. add more tests.. #shame + +def _get_good_connection(): + connections = {} + connections["streams_publishes"] = [ + {"name" : "test_n", + "location" : "test_l", + "client_role" : "test_cr", + "type" : "message_router", + "config_key" : "test_c", + "aaf_username": "test_u", + "aaf_password": "test_p" + }, + {"name" : "test_n2", + "location" : "test_l", + "client_role" : "test_cr", + "type" : "message_router", + "config_key" : "test_c", + "aaf_username": "test_u", + "aaf_password": "test_p" + }, + {"name" : "test_feed00", + "location" : "test_l", + "type" : "data_router", + "config_key" : "mydrconfigkey" + } + ] + connections["streams_subscribes"] = [ + {"name" : "test_n", + "location" : "test_l", + "client_role" : "test_cr", + "type" : "message_router", + "config_key" : "test_c", + "aaf_username": "test_u", + "aaf_password": "test_p" + }, + {"name" : "test_n2", + "location" : "test_l", + "client_role" : "test_cr", + "type" : "message_router", + "config_key" : "test_c", + "aaf_username": "test_u", + "aaf_password": "test_p" + } + ] + return connections + +def test_validate_cons(): + #test good streams + good_conn = _get_good_connection() + _validate_conns(good_conn) + + #mutate + nosub = _get_good_connection().pop("streams_subscribes") + with pytest.raises(BadConnections) as excinfo: + _validate_conns(nosub) + + nopub = _get_good_connection().pop("streams_publishes") + with pytest.raises(BadConnections) as excinfo: + _validate_conns(nopub) + + noloc = _get_good_connection()["streams_publishes"][0].pop("location") + with pytest.raises(BadConnections) as excinfo: + _validate_conns(noloc) + diff --git a/cdap/cdapplugin/tox.ini b/cdap/cdapplugin/tox.ini new file mode 100644 index 0000000..afabca4 --- /dev/null +++ b/cdap/cdapplugin/tox.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py27 +[testenv] +deps= + pytest + uuid==1.30 + cloudify==3.4 +commands=pytest diff --git a/cdap/demo_blueprints/cdap_hello_world.yaml b/cdap/demo_blueprints/cdap_hello_world.yaml new file mode 100644 index 0000000..1b7ff90 --- /dev/null +++ b/cdap/demo_blueprints/cdap_hello_world.yaml @@ -0,0 +1,40 @@ +tosca_definitions_version: cloudify_dsl_1_3 + +imports: + - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.0.2/cdap_types.yaml + +inputs: + hello_world_jar_url: + type: string + connected_broker_dns_name: + type: string + default : "cdap_broker" + +node_templates: + + hw_cdap_app: + type: dcae.nodes.MicroService.cdap + properties: + service_component_type: + 'hello_world' + jar_url: { get_input : hello_world_jar_url } + artifact_name: "HelloWorld" + artifact_version: "3.4.3" + namespace: "cloudifyhwtest" + programs: + [{"program_type" : "flows", "program_id" : "WhoFlow"}, {"program_type" : "services", "program_id" : "Greeting"}] + streamname: + 'who' + service_endpoints: + [{"service_name" : "Greeting", "service_endpoint" : "greet", "endpoint_method" : "GET"}] + interfaces: + cloudify.interfaces.lifecycle: + create: + inputs: + connected_broker_dns_name: { get_input: connected_broker_dns_name } + +outputs: + hw_cdap_app_name: + value: + {get_attribute:[hw_cdap_app, service_component_name]} diff --git a/cdap/demo_blueprints/cdap_hello_world_reconfigure.sh b/cdap/demo_blueprints/cdap_hello_world_reconfigure.sh new file mode 100755 index 0000000..c5df2f5 --- /dev/null +++ b/cdap/demo_blueprints/cdap_hello_world_reconfigure.sh @@ -0,0 +1,4 @@ +#!/bin/bash +cfy executions start -d cdap-hello-world -w execute_operation -p '{"operation" : "reconfiguration.app_config_reconfigure", "node_ids" : ["hw_cdap_app"], "operation_kwargs" : {"new_config_template" : {"foo":"bar"}}, "allow_kwargs_override": true}' +cfy executions start -d cdap-hello-world -w execute_operation -p '{"operation" : "reconfiguration.app_preferences_reconfigure", "node_ids" : ["hw_cdap_app"], "operation_kwargs" : {"new_config_template" : {"fooprefs":"barprefs"}}, "allow_kwargs_override": true}' +cfy executions start -d cdap-hello-world -w execute_operation -p '{"operation" : "reconfiguration.app_smart_reconfigure", "node_ids" : ["hw_cdap_app"], "operation_kwargs" : {"new_config_template" : {"fooprefs":"SO SMARTTTTTT", "foo":"SO SMART AGAINNNNN"}}, "allow_kwargs_override": true}' diff --git a/cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml b/cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml new file mode 100644 index 0000000..ea02543 --- /dev/null +++ b/cdap/demo_blueprints/cdap_hello_world_with_dmaap.yaml @@ -0,0 +1,148 @@ +tosca_definitions_version: cloudify_dsl_1_3 + +imports: + - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.0.2/cdap_types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/dmaap/1.1.0/dmaap.yaml + +inputs: + hello_world_jar_url: + type: string + connected_broker_dns_name: + type: string + default : "cdap_broker" + + #aaf inputs + client_role: + type: string + topic00fqtn: + type: string + topic01fqtn: + type: string + aafu0: + type: string + default: "foo0" + aafp0: + type: string + default: "bar0" + aafu1: + type: string + default : "foo1" + aafp1: + type: string + default : "bar1" + aafu2: + type: string + default: "foo2" + aafp2: + type: string + default: "bar2" + aafu3: + type: string + default : "foo3" + aafp3: + type: string + default : "bar3" + +node_templates: + topic00: + type: dcae.nodes.ExistingTopic + properties: + fqtn: { get_input : topic00fqtn } + topic01: + type: dcae.nodes.ExistingTopic + properties: + fqtn: { get_input : topic01fqtn } + feed00: + type: dcae.nodes.Feed + properties: + feed_name: "FEEDME-12" + feed_description: "Tommy Test feed for CDAP Publishes" + feed_version: 6.6.6 + aspr_classification: "unclassified" + + hw_cdap_app: + type: dcae.nodes.MicroService.cdap + properties: + service_component_type: 'hello_world' + jar_url: { get_input : hello_world_jar_url } + artifact_name: "HelloWorld" + artifact_version: "3.4.3" + namespace: "cloudifyhwtest" + programs: + [{"program_type" : "flows", "program_id" : "WhoFlow"}, {"program_type" : "services", "program_id" : "Greeting"}] + streamname: + 'who' + service_endpoints: + [{"service_name" : "Greeting", "service_endpoint" : "greet", "endpoint_method" : "GET"}] + + #special key for CDAP plugin + connections: + streams_publishes: + - name: topic00 #MR pub 1 + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey0" + aaf_username: { get_input: aafu0 } + aaf_password: { get_input: aafp0 } + - name: topic01 #MR pub 2 + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey1" + aaf_username: { get_input: aafu1 } + aaf_password: { get_input: aafp1 } + - name: feed00 #Feed pub 1 + location: mtc5 + type: data_router + config_key: "mydrconfigkey" + streams_subscribes: + - name: topic00 #MEANT FOR DEMO ONLY! Subscribing and publishing to same topic. Not real example. + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey2" + aaf_username: { get_input: aafu2 } + aaf_password: { get_input: aafp2 } + - name: topic01 + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey3" + aaf_username: { get_input: aafu3 } + aaf_password: { get_input: aafp3 } + + relationships: + - type: dcae.relationships.publish_events + target: topic00 #MEANT FOR DEMO ONLY! Subscribing and publishing to same topic. Not real example. + - type: dcae.relationships.publish_events + target: topic01 + - type: dcae.relationships.subscribe_to_events + target: topic00 + - type: dcae.relationships.subscribe_to_events + target: topic01 + - type: dcae.relationships.publish_files + target: feed00 + + interfaces: + cloudify.interfaces.lifecycle: + create: + inputs: + connected_broker_dns_name: { get_input: connected_broker_dns_name } + +outputs: + hw_cdap_app_name: + value: {get_attribute:[hw_cdap_app, service_component_name]} + + topic00_data: + description: "Topic 00 data" + value: { get_attribute: [hw_cdap_app, topic00]} + + topic01_data: + description: "Topic 01 data" + value: { get_attribute: [hw_cdap_app, topic01]} + + + + diff --git a/cdap/demo_blueprints/cdap_hello_world_with_laika.yaml b/cdap/demo_blueprints/cdap_hello_world_with_laika.yaml new file mode 100644 index 0000000..4587a47 --- /dev/null +++ b/cdap/demo_blueprints/cdap_hello_world_with_laika.yaml @@ -0,0 +1,80 @@ +tosca_definitions_version: cloudify_dsl_1_3 + +imports: + - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.0.2/cdap_types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/docker/2.1.0/node-type.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/relationship/1.0.0/node-type.yaml + +inputs: + hello_world_jar_url: + type: string + laika_image: + type: string + connected_broker_dns_name: + type: string + default : "cdap_broker" + +node_templates: + + hw_cdap_app: + type: dcae.nodes.MicroService.cdap + properties: + service_component_type: + 'hello_world' + jar_url: { get_input : hello_world_jar_url } + artifact_name: "HelloWorld" + artifact_version: "3.4.3" + namespace: "cloudifyhwtest" + programs: + [{"program_type" : "flows", "program_id" : "WhoFlow"}, {"program_type" : "services", "program_id" : "Greeting"}] + streamname: + 'who' + service_endpoints: + [{"service_name" : "Greeting", "service_endpoint" : "greet", "endpoint_method" : "GET"}] + + connections: + services_calls: + - service_component_type: laika + config_key: "laika_handle" + + relationships: + - type: dcae.relationships.component_connected_to + target: laika-one + + interfaces: + cloudify.interfaces.lifecycle: + create: + inputs: + connected_broker_dns_name: { get_input: connected_broker_dns_name } + + laika-one: + type: dcae.nodes.DockerContainerForComponents + properties: + service_component_type: 'laika' + service_id: 'this_is_dumb' + location_id: 'this_is_dumb' + image: { get_input : laika_image } + # Trying without health check + relationships: + - type: dcae.relationships.component_contained_in + target: docker_host + interfaces: + cloudify.interfaces.lifecycle: + stop: + inputs: + cleanup_image: + False + + docker_host: + type: dcae.nodes.SelectedDockerHost + properties: + location_id: 'this is dumb' + docker_host_override: 'platform_dockerhost' + +outputs: + hw_cdap_app_name: + value: {get_attribute:[hw_cdap_app, service_component_name]} + + + diff --git a/cdap/demo_blueprints/cdap_hello_world_with_mr.yaml b/cdap/demo_blueprints/cdap_hello_world_with_mr.yaml new file mode 100644 index 0000000..e1e0adb --- /dev/null +++ b/cdap/demo_blueprints/cdap_hello_world_with_mr.yaml @@ -0,0 +1,134 @@ +tosca_definitions_version: cloudify_dsl_1_3 + +imports: + - http://www.getcloudify.org/spec/cloudify/3.4/types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/cdap/14.0.2/cdap_types.yaml + - {{ ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2 }}/type_files/dmaap/1.1.0/dmaap.yaml + +inputs: + hello_world_jar_url: + type: string + connected_broker_dns_name: + type: string + default : "cdap_broker" + + #aaf inputs + client_role: + type: string + topic00fqtn: + type: string + topic01fqtn: + type: string + aafu0: + type: string + default: "foo0" + aafp0: + type: string + default: "bar0" + aafu1: + type: string + default : "foo1" + aafp1: + type: string + default : "bar1" + aafu2: + type: string + default: "foo2" + aafp2: + type: string + default: "bar2" + aafu3: + type: string + default : "foo3" + aafp3: + type: string + default : "bar3" + +node_templates: + topic00: + type: dcae.nodes.ExistingTopic + properties: + fqtn: { get_input : topic00fqtn } + + topic01: + type: dcae.nodes.ExistingTopic + properties: + fqtn: { get_input : topic01fqtn } + + hw_cdap_app: + type: dcae.nodes.MicroService.cdap + properties: + service_component_type: + 'hello_world' + jar_url: { get_input : hello_world_jar_url } + artifact_name: "HelloWorld" + artifact_version: "3.4.3" + namespace: "cloudifyhwtest" + programs: + [{"program_type" : "flows", "program_id" : "WhoFlow"}, {"program_type" : "services", "program_id" : "Greeting"}] + streamname: + 'who' + service_endpoints: + [{"service_name" : "Greeting", "service_endpoint" : "greet", "endpoint_method" : "GET"}] + + #special key for CDAP plugin + connections: + streams_publishes: + - name: topic00 #MR pub 1 + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey0" + aaf_username: { get_input: aafu0 } + aaf_password: { get_input: aafp0 } + - name: topic01 #MR pub 2 + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey1" + aaf_username: { get_input: aafu1 } + aaf_password: { get_input: aafp1 } + streams_subscribes: + - name: topic00 #MEANT FOR DEMO ONLY! Subscribing and publishing to same topic. Not real example. + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey2" + aaf_username: { get_input: aafu2 } + aaf_password: { get_input: aafp2 } + - name: topic01 + location: mtc5 + client_role: { get_input: client_role } + type: message_router + config_key: "myconfigkey3" + aaf_username: { get_input: aafu3 } + aaf_password: { get_input: aafp3 } + + relationships: + - type: dcae.relationships.publish_events + target: topic00 #MEANT FOR DEMO ONLY! Subscribing and publishing to same topic. Not real example. + - type: dcae.relationships.publish_events + target: topic01 + - type: dcae.relationships.subscribe_to_events + target: topic00 + - type: dcae.relationships.subscribe_to_events + target: topic01 + + interfaces: + cloudify.interfaces.lifecycle: + create: + inputs: + connected_broker_dns_name: { get_input: connected_broker_dns_name } + +outputs: + hw_cdap_app_name: + value: {get_attribute:[hw_cdap_app, service_component_name]} + + topic00_data: + description: "Topic 00 data" + value: { get_attribute: [hw_cdap_app, topic00]} + + topic01_data: + description: "Topic 01 data" + value: { get_attribute: [hw_cdap_app, topic01]} + diff --git a/cdap/pom.xml b/cdap/pom.xml new file mode 100644 index 0000000..d31c06b --- /dev/null +++ b/cdap/pom.xml @@ -0,0 +1,283 @@ + + + 4.0.0 + + + org.onap.dcaegen2.platform + plugins + 1.0.0-SNAPSHOT + + org.onap.dcaegen2.platform.plugins + cdap + cdap-plugin + 1.0.0-SNAPSHOT + http://maven.apache.org + + UTF-8 + . + + http://localhost:9000 + + + py + Python + **/*.py + + + + + + ${project.artifactId}-${project.version} + + + + org.codehaus.mojo + sonar-maven-plugin + 2.7.1 + + + + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.6 + + true + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + true + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + default-jar + + + + + + + + org.apache.maven.plugins + maven-install-plugin + 2.4 + + true + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.4 + + true + + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2.1 + + + clean phase script + clean + exec + + ${session.executionRootDirectory}/mvn-phase-script.sh + + ${project.artifactId} + clean + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + + + + + + + generate-sources script + generate-sources + exec + + mvn-phase-script.sh + + ${project.artifactId} + generate-sources + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + + + + + + + compile script + compile + exec + + mvn-phase-script.sh + + ${project.artifactId} + compile + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + + + + + + + package script + package + exec + + mvn-phase-script.sh + + ${project.artifactId} + package + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + + + + + + + test script + test + exec + + mvn-phase-script.sh + + ${project.artifactId} + test + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + + + + + + + install script + install + exec + + mvn-phase-script.sh + + ${project.artifactId} + install + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + + + + + + + deploy script + deploy + exec + + mvn-phase-script.sh + + ${project.artifactId} + deploy + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${onap.nexus.url} + + + + + + + + + diff --git a/mvn-phase-script.sh b/mvn-phase-script.sh new file mode 100755 index 0000000..bf761dc --- /dev/null +++ b/mvn-phase-script.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +echo "running script: [$0] for module [$1] at stage [$2]" + +MVN_MODULE=$1 + +echo "=> Prepare environment " +#env +if [ -z "$MVN_DOCKERREG_URL" ]; then + MVN_DOCKERREG_URL='nexus3.onap.org:10001' +fi +if [ -z "$SETTINGS_FILE" ]; then + SETTINGS_FILE='settings.xml' +fi + + +TIMESTAMP=$(date +%C%y%m%dT%H%M%S) +export BUILD_NUMBER="${TIMESTAMP}" + +# expected environment variables +if [ -z "${MVN_NEXUSPROXY}" ]; then + echo "MVN_NEXUSPROXY environment variable not set. Cannot proceed" + exit +fi +MVN_NEXUSPROXY_HOST=$(echo $MVN_NEXUSPROXY |cut -f3 -d'/' | cut -f1 -d':') + + +if [ -z "${SETTINGS_FILE}" ]; then + echo "SETTINGS_FILE environment variable not set. Cannot proceed" + exit +fi + +if [ ]; then + +# login to all docker registries +DOCKER_REPOSITORIES="nexus3.onap.org:10001 nexus3.onap.org:10002 nexus3.onap.org:10003 nexus3.onap.org:10004" +for DOCKER_REPOSITORY in $DOCKER_REPOSITORIES; +do + USER=$(xpath -e "//servers/server[id='$DOCKER_REPOSITORY']/username/text()" "$SETTINGS_FILE") + PASS=$(xpath -e "//servers/server[id='$DOCKER_REPOSITORY']/password/text()" "$SETTINGS_FILE") + + if [ -z "$USER" ]; then + echo "Error: no user provided" + fi + + if [ -z "$PASS" ]; then + echo "Error: no password provided" + fi + + [ -z "$PASS" ] && PASS_PROVIDED="" || PASS_PROVIDED="" + echo docker login $DOCKER_REPOSITORY -u "$USER" -p "$PASS_PROVIDED" + docker login $DOCKER_REPOSITORY -u "$USER" -p "$PASS" +done +fi + +# ste up env variables, get ready for template resolution +export ONAPTEMPLATE_RAWREPOURL_org_onap_dcaegen2="${MVN_NEXUSPROXY}/content/sites/raw" +export ONAPTEMPLATE_PYPIURL_org_onap_dcaegen2="${MVN_NEXUSPROXY}/content/sites/pypi" +export ONAPTEMPLATE_DOCKERREGURL_org_onap_dcaegen2="${MVN_DOCKERREG_URL}" +export ONAPTEMPLATE_GITREPO_org_onap_dcaegen2="https://gerrit.onap.org" + +# use the version text detect which phase we are in in LF CICD process: verify, merge, or (daily) release +LF_PHASE="verify" + +#case "$phase" in +# verify|merge ) REPO="${NEXUS_RAW}/org.openecomp.dcae.pgaas/deb-snapshots" ;; +# release ) REPO="${NEXUS_RAW}/org.openecomp.dcae.pgaas/deb-releases" ;; +#esac + +# mvn phase in life cycle +MVN_PHASE="$2" + +case $MVN_PHASE in +clean) + echo "==> clean phase script" + ;; +generate-sources) + echo "==> generate-sources phase script. Dir is $(pwd)" + + TEMPLATES=$(env |grep ONAPTEMPLATE) + echo "====> Resolving the following temaplate from environment variables " + echo "[$TEMPLATES]" + SELFFILE=$(echo $0 | rev | cut -f1 -d '/' | rev) + for TEMPLATE in $TEMPLATES; do + KEY=$(echo $TEMPLATE | cut -f1 -d'=') + VALUE=$(echo $TEMPLATE | cut -f2 -d'=') + VALUE2=$(echo $TEMPLATE | cut -f2 -d'=' |sed 's/\//\\\//g') + FILES=$(grep -rl "$KEY") + + # assuming FILES is not longer than 2M bytes, the limit for variable value max size on this VM + for F in $FILES; do + if [[ $F == *"$SELFFILE" ]]; then + continue + fi + echo "======> Resolving template $KEY to value $VALUE for file $F" + sed -i "s/{{[[:space:]]*$KEY[[:space:]]*}}/$VALUE2/g" $F + done + + #if [ ! -z "$FILES" ]; then + # echo "====> Resolving template $VALUE to value $VALUE" + # #CMD="grep -rl \"$VALUE\" | tr '\n' '\0' | xargs -0 sed -i \"s/{{[[:space:]]*$VALUE[[:space:]]*}}/$VALUE/g\"" + # grep -rl "$KEY" | tr '\n' '\0' | xargs -0 sed -i 's/$KEY/$VALUE2/g' + # #echo $CMD + # #eval $CMD + #fi + done + echo "====> Done template reolving" + ;; +compile) + echo "==> compile phase script" + ;; +test) + echo "==> test phase script" + ;; +package) + echo "==> package phase script" + ;; +install) + echo "==> install phase script" + ;; +deploy) + echo "==> deploy phase script" + + # prepare credential for curl use (raw repo) + USER=$(xpath -q -e "//servers/server[id='ecomp-raw']/username/text()" "$SETTINGS_FILE") + PASS=$(xpath -q -e "//servers/server[id='ecomp-raw']/password/text()" "$SETTINGS_FILE") + export NETRC=$(mktemp) + echo "machine $MVN_NEXUSPROXY_HOST login ${USER} password ${PASS}" >> "${NETRC}" + set -x; curl -k --netrc-file '${NETRC}' --upload-file '{0}' '${REPO}/{2}-{1}' + + + + # login to all docker registries + USER=$(xpath -e "//servers/server[id='$MVN_DOCKERREG_URL']/username/text()" "$SETTINGS_FILE") + PASS=$(xpath -e "//servers/server[id='$MVN_DOCKERREG_URL']/password/text()" "$SETTINGS_FILE") + if [ -z "$USER" ]; then + echo "Error: no user provided" + fi + if [ -z "$PASS" ]; then + echo "Error: no password provided" + fi + [ -z "$PASS" ] && PASS_PROVIDED="" || PASS_PROVIDED="" + echo docker login $MVN_DOCKERREG_URL -u "$USER" -p "$PASS_PROVIDED" + docker login $MVN_DOCKERREG_URL -u "$USER" -p "$PASS" + + #docker push + ;; +*) + echo "==> unprocessed phase" + ;; +esac + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..52043ff --- /dev/null +++ b/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + org.onap.oparent + oparent + 1.0.0-SNAPSHOT + + + org.onap.dcaegen2.platform + plugins + plugins + 1.0.0-SNAPSHOT + http://maven.apache.org + pom + + cdap + + + -- cgit 1.2.3-korg