From 06bdb6a3ec916137534e627a275141c8c587dff7 Mon Sep 17 00:00:00 2001 From: Krzysztof Bijakowski Date: Tue, 5 Sep 2017 10:26:20 +0200 Subject: Cloudify blueprint for provisioning ONAP Change-Id: Ibe0f7e626ca2ebaa5d5e7eeb49a33b19cacbafa3 Issue-ID: OOM-106 Signed-off-by: Krzysztof Bijakowski --- .../onap/configure_docker_secret_workaround.py | 40 +++++++ cloudify/scripts/onap/create_init_pod.py | 65 ++++++++++ cloudify/scripts/onap/create_namespace.py | 101 ++++++++++++++++ cloudify/scripts/onap/create_resources_services.py | 131 ++++++++++++++++++++ cloudify/scripts/onap/delete_init_pod.py | 64 ++++++++++ cloudify/scripts/onap/delete_namespace.py | 101 ++++++++++++++++ cloudify/scripts/onap/delete_resources_services.py | 132 +++++++++++++++++++++ cloudify/scripts/onap/patch_definitions.py | 1 - cloudify/scripts/onap/provision_definitions.py | 1 - cloudify/scripts/onap/read_definitions.py | 1 - cloudify/types/onap.yaml | 61 +++++++++- 11 files changed, 692 insertions(+), 6 deletions(-) create mode 100644 cloudify/scripts/onap/configure_docker_secret_workaround.py create mode 100644 cloudify/scripts/onap/create_init_pod.py create mode 100644 cloudify/scripts/onap/create_namespace.py create mode 100644 cloudify/scripts/onap/create_resources_services.py create mode 100644 cloudify/scripts/onap/delete_init_pod.py create mode 100644 cloudify/scripts/onap/delete_namespace.py create mode 100644 cloudify/scripts/onap/delete_resources_services.py delete mode 100644 cloudify/scripts/onap/patch_definitions.py delete mode 100644 cloudify/scripts/onap/provision_definitions.py delete mode 100644 cloudify/scripts/onap/read_definitions.py (limited to 'cloudify') diff --git a/cloudify/scripts/onap/configure_docker_secret_workaround.py b/cloudify/scripts/onap/configure_docker_secret_workaround.py new file mode 100644 index 0000000000..6e9deff059 --- /dev/null +++ b/cloudify/scripts/onap/configure_docker_secret_workaround.py @@ -0,0 +1,40 @@ +from fabric.api import run + +from cloudify import ctx +from cloudify.exceptions import NonRecoverableError + + +def _retrieve_namespace(): + namespace = ctx.node.properties.get( + 'namespace', + ctx.node.properties + .get('options', {}) + .get('namespace', None) + ) + + if not namespace: + raise NonRecoverableError( + 'Namespace is not defined (node={})'.format(ctx.node.name) + ) + + return namespace + + +def configure_secret(): + namespace = _retrieve_namespace() + ctx.logger.info( + 'Configuring docker secrets for namespace: {0}'.format(namespace) + ) + + command = 'kubectl create secret ' \ + 'docker-registry onap-docker-registry-key ' \ + '--docker-server=nexus3.onap.org:10001 ' \ + '--docker-username=docker ' \ + '--docker-password=docker ' \ + '--docker-email=email@email.com ' \ + '--namespace={0}'.format(namespace) + + ctx.logger.info('Command "{0}" will be executed'.format(command)) + run(command) + + ctx.logger.info('Docker secrets configured successfully') diff --git a/cloudify/scripts/onap/create_init_pod.py b/cloudify/scripts/onap/create_init_pod.py new file mode 100644 index 0000000000..c82172d15f --- /dev/null +++ b/cloudify/scripts/onap/create_init_pod.py @@ -0,0 +1,65 @@ +import pip + +from cloudify import ctx +from cloudify.exceptions import NonRecoverableError + + +SERVICES_FILE_PARTS_SEPARATOR = '---' + + +def _import_or_install(): + try: + import yaml + except ImportError: + pip.main(["install", "pyaml"]) + + try: + import cloudify_kubernetes.tasks as kubernetes_plugin + except ImportError: + pip.main([ + "install", + "https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/archive/1.2.1rc1.zip" + ]) + + import yaml + import cloudify_kubernetes.tasks as kubernetes_plugin + + return yaml, kubernetes_plugin + + +def _retrieve_path(): + return ctx.node.properties.get('init_pod', None) + + +def _save_deployment_result(key): + result = ctx.instance.runtime_properties['kubernetes'] + ctx.instance.runtime_properties[key] = result + ctx.instance.runtime_properties['kubernetes'] = {} + + +def _do_create_init_pod(kubernetes_plugin, yaml): + ctx.logger.info('Creating init pod') + init_pod_file_path = _retrieve_path() + + if not init_pod_file_path: + raise NonRecoverableError('Init pod file is not defined.') + + temp_file_path = ctx.download_resource_and_render( + init_pod_file_path + ) + + with open(temp_file_path) as temp_file: + init_pod_file_content = temp_file.read() + init_pod_yaml_content = yaml.load(init_pod_file_content) + + kubernetes_plugin.resource_create(definition=init_pod_yaml_content) + _save_deployment_result('init_pod') + + ctx.logger.info('Init pod created successfully') + + +if __name__ == '__main__': + yaml, kubernetes_plugin = _import_or_install() + + _do_create_init_pod(kubernetes_plugin, yaml) + diff --git a/cloudify/scripts/onap/create_namespace.py b/cloudify/scripts/onap/create_namespace.py new file mode 100644 index 0000000000..c0f1f19680 --- /dev/null +++ b/cloudify/scripts/onap/create_namespace.py @@ -0,0 +1,101 @@ +import pip + +from cloudify import ctx +from cloudify.exceptions import NonRecoverableError + + +def _import_or_install(): + try: + import yaml + except ImportError: + pip.main(["install", "pyaml"]) + + try: + import cloudify_kubernetes.tasks as kubernetes_plugin + except ImportError: + pip.main([ + "install", + "https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/archive/1.2.1rc1.zip" + ]) + + import yaml + import cloudify_kubernetes.tasks as kubernetes_plugin + + return yaml, kubernetes_plugin + + +def _retrieve_namespace(): + namespace = ctx.node.properties.get( + 'namespace', + ctx.node.properties + .get('options', {}) + .get('namespace', None) + ) + + if not namespace: + raise NonRecoverableError( + 'Namespace is not defined (node={})'.format(ctx.node.name) + ) + + return namespace + + +def _prepare_namespace_resource_template(name): + return { + 'definition': { + 'apiVersion': 'v1', + 'kind': 'Namespace', + 'metadata': { + 'name': name, + 'labels': { + 'name': name + }, + }, + }, + 'api_mapping': { + 'create': { + 'api': 'CoreV1Api', + 'method': 'create_namespace', + 'payload': 'V1Namespace' + }, + 'read': { + 'api': 'CoreV1Api', + 'method': 'read_namespace', + }, + 'delete': { + 'api': 'CoreV1Api', + 'method': 'delete_namespace', + 'payload': 'V1DeleteOptions' + } + } + } + + +def _save_deployment_result(key): + result = ctx.instance.runtime_properties['kubernetes'] + ctx.instance.runtime_properties[key] = result + ctx.instance.runtime_properties['kubernetes'] = {} + + +def _do_create_namespace(kubernetes_plugin): + namespace = _retrieve_namespace() + ctx.logger.info('Creating namespace: {0}'.format(namespace)) + + namespace_resource_template = _prepare_namespace_resource_template( + namespace + ) + + ctx.logger.debug( + 'Kubernetes object which will be deployed: {0}' + .format(namespace_resource_template) + ) + + kubernetes_plugin.custom_resource_create(**namespace_resource_template) + _save_deployment_result('namespace') + ctx.logger.info('Namespace created successfully') + + +if __name__ == '__main__': + _, kubernetes_plugin = _import_or_install() + + _do_create_namespace(kubernetes_plugin) diff --git a/cloudify/scripts/onap/create_resources_services.py b/cloudify/scripts/onap/create_resources_services.py new file mode 100644 index 0000000000..8548e29b70 --- /dev/null +++ b/cloudify/scripts/onap/create_resources_services.py @@ -0,0 +1,131 @@ +import pip + +from cloudify import ctx + + +SERVICES_FILE_PARTS_SEPARATOR = '---' + + +def _import_or_install(): + try: + import yaml + except ImportError: + pip.main(["install", "pyaml"]) + + try: + import cloudify_kubernetes.tasks as kubernetes_plugin + except ImportError: + pip.main([ + "install", + "https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/archive/1.2.1rc1.zip" + ]) + + try: + import jinja2 + except ImportError: + pip.main(["install", "jinja2"]) + + import yaml + import jinja2 + import cloudify_kubernetes.tasks as kubernetes_plugin + + return yaml, kubernetes_plugin, jinja2 + + +def _init_jinja(jinja2): + return jinja2.Environment( + loader=jinja2.BaseLoader() + ) + + +def _render_template(jinja_env, template_content, values): + template_content = template_content.replace('.Values', 'Values') + + template = jinja_env.from_string(template_content) + rendered_template = template.render(Values=values) + return rendered_template + + +def _retrieve_resources_paths(): + return ctx.node.properties.get('resources', []) + + +def _retrieve_services_paths(): + return ctx.node.properties.get('services', None) + + +def _retrieve_values(yaml): + values_file_path = ctx.node.properties.get('values', None) + + if values_file_path: + return yaml.load(ctx.get_resource(values_file_path)) + + ctx.logger.warn('Values file not found') + + +def _save_deployment_result(key): + result = ctx.instance.runtime_properties['kubernetes'] + ctx.instance.runtime_properties[key] = result + ctx.instance.runtime_properties['kubernetes'] = {} + + +def _do_create_resources(kubernetes_plugin, yaml, jinja_env, values): + for path in _retrieve_resources_paths(): + ctx.logger.info('Creating resource defined in: {0}'.format(path)) + + template_content = ctx.get_resource(path) + yaml_content = _render_template( + jinja_env, + template_content, + values + ) + content = yaml.load(yaml_content) + + kubernetes_plugin.resource_create(definition=content) + _save_deployment_result( + 'resource_{0}'.format(content['metadata']['name']) + ) + + ctx.logger.info('Resources created successfully') + + +def _do_create_services(kubernetes_plugin, yaml, jinja_env, values): + ctx.logger.info('Creating services') + services_file_path = _retrieve_services_paths() + + if not services_file_path: + ctx.logger.warn( + 'Service file is not defined. Skipping services provisioning !' + ) + + return + + template_content = ctx.get_resource(services_file_path) + yaml_content = _render_template( + jinja_env, + template_content, + values + ) + + yaml_content_parts = \ + yaml_content.split(SERVICES_FILE_PARTS_SEPARATOR) + + for yaml_content_part in yaml_content_parts: + content = yaml.load(yaml_content_part) + + kubernetes_plugin.resource_create(definition=content) + _save_deployment_result( + 'service_{0}'.format(content['metadata']['name']) + ) + + ctx.logger.info('Services created successfully') + + +if __name__ == '__main__': + yaml, kubernetes_plugin, jinja2 = _import_or_install() + jinja_env = _init_jinja(jinja2) + values = _retrieve_values(yaml) + + _do_create_resources(kubernetes_plugin, yaml, jinja_env, values) + _do_create_services(kubernetes_plugin, yaml, jinja_env, values) + diff --git a/cloudify/scripts/onap/delete_init_pod.py b/cloudify/scripts/onap/delete_init_pod.py new file mode 100644 index 0000000000..1da805b959 --- /dev/null +++ b/cloudify/scripts/onap/delete_init_pod.py @@ -0,0 +1,64 @@ +import pip + +from cloudify import ctx +from cloudify.exceptions import NonRecoverableError + + +SERVICES_FILE_PARTS_SEPARATOR = '---' + + +def _import_or_install(): + try: + import yaml + except ImportError: + pip.main(["install", "pyaml"]) + + try: + import cloudify_kubernetes.tasks as kubernetes_plugin + except ImportError: + pip.main([ + "install", + "https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/archive/1.2.1rc1.zip" + ]) + + import yaml + import cloudify_kubernetes.tasks as kubernetes_plugin + + return yaml, kubernetes_plugin + + +def _retrieve_path(): + return ctx.node.properties.get('init_pod', None) + + +def _set_deployment_result(key): + result = ctx.instance.runtime_properties.pop(key) + ctx.instance.runtime_properties['kubernetes'] = result + + +def _do_delete_init_pod(kubernetes_plugin, yaml): + ctx.logger.info('Deleting init pod') + init_pod_file_path = _retrieve_path() + + if not init_pod_file_path: + raise NonRecoverableError('Init pod file is not defined.') + + temp_file_path = ctx.download_resource_and_render( + init_pod_file_path + ) + + with open(temp_file_path) as temp_file: + init_pod_file_content = temp_file.read() + init_pod_yaml_content = yaml.load(init_pod_file_content) + + _set_deployment_result('init_pod') + kubernetes_plugin.resource_delete(definition=init_pod_yaml_content) + + ctx.logger.info('Init pod deleted successfully') + + +if __name__ == '__main__': + yaml, kubernetes_plugin = _import_or_install() + + _do_delete_init_pod(kubernetes_plugin, yaml) + diff --git a/cloudify/scripts/onap/delete_namespace.py b/cloudify/scripts/onap/delete_namespace.py new file mode 100644 index 0000000000..6973e59944 --- /dev/null +++ b/cloudify/scripts/onap/delete_namespace.py @@ -0,0 +1,101 @@ +import pip + +from cloudify import ctx +from cloudify.exceptions import NonRecoverableError + + +def _import_or_install(): + try: + import yaml + except ImportError: + pip.main(["install", "pyaml"]) + + try: + import cloudify_kubernetes.tasks as kubernetes_plugin + except ImportError: + pip.main([ + "install", + "https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/archive/1.2.1rc1.zip" + ]) + + import yaml + import cloudify_kubernetes.tasks as kubernetes_plugin + + return yaml, kubernetes_plugin + + +def _retrieve_namespace(): + namespace = ctx.node.properties.get( + 'namespace', + ctx.node.properties + .get('options', {}) + .get('namespace', None) + ) + + if not namespace: + raise NonRecoverableError( + 'Namespace is not defined (node={})'.format(ctx.node.name) + ) + + return namespace + + +def _prepare_namespace_resource_template(name): + return { + 'definition': { + 'apiVersion': 'v1', + 'kind': 'Namespace', + 'metadata': { + 'name': name, + 'labels': { + 'name': name + }, + }, + }, + 'api_mapping': { + 'create': { + 'api': 'CoreV1Api', + 'method': 'create_namespace', + 'payload': 'V1Namespace' + }, + 'read': { + 'api': 'CoreV1Api', + 'method': 'read_namespace', + }, + 'delete': { + 'api': 'CoreV1Api', + 'method': 'delete_namespace', + 'payload': 'V1DeleteOptions' + } + } + } + + +def _set_deployment_result(key): + result = ctx.instance.runtime_properties.pop(key) + ctx.instance.runtime_properties['kubernetes'] = result + + +def _do_delete_namespace(kubernetes_plugin): + namespace = _retrieve_namespace() + ctx.logger.info('Deleting namespace: {0}'.format(namespace)) + + namespace_resource_template = _prepare_namespace_resource_template( + namespace + ) + + ctx.logger.debug( + 'Kubernetes object which will be deleted: {0}' + .format(namespace_resource_template) + ) + + _set_deployment_result('namespace') + kubernetes_plugin.custom_resource_delete(**namespace_resource_template) + ctx.logger.info('Namespace deleted successfully') + + +if __name__ == '__main__': + _, kubernetes_plugin = _import_or_install() + + _do_delete_namespace(kubernetes_plugin) + diff --git a/cloudify/scripts/onap/delete_resources_services.py b/cloudify/scripts/onap/delete_resources_services.py new file mode 100644 index 0000000000..305a7484bd --- /dev/null +++ b/cloudify/scripts/onap/delete_resources_services.py @@ -0,0 +1,132 @@ +import pip + +from cloudify import ctx +from cloudify.exceptions import NonRecoverableError + + +SERVICES_FILE_PARTS_SEPARATOR = '---' + + +def _import_or_install(): + try: + import yaml + except ImportError: + pip.main(["install", "pyaml"]) + + try: + import cloudify_kubernetes.tasks as kubernetes_plugin + except ImportError: + pip.main([ + "install", + "https://github.com/cloudify-incubator/cloudify-kubernetes-plugin/archive/1.2.1rc1.zip" + ]) + + try: + import jinja2 + except ImportError: + pip.main(["install", "jinja2"]) + + import yaml + import jinja2 + import cloudify_kubernetes.tasks as kubernetes_plugin + + return yaml, kubernetes_plugin, jinja2 + + +def _init_jinja(jinja2): + return jinja2.Environment( + loader=jinja2.BaseLoader() + ) + + +def _render_template(jinja_env, template_content, values): + template_content = template_content.replace('.Values', 'Values') + + template = jinja_env.from_string(template_content) + rendered_template = template.render(Values=values) + return rendered_template + + +def _retrieve_resources_paths(): + return ctx.node.properties.get('resources', []) + + +def _retrieve_services_paths(): + return ctx.node.properties.get('services', None) + + +def _retrieve_values(yaml): + values_file_path = ctx.node.properties.get('values', None) + + if values_file_path: + return yaml.load(ctx.get_resource(values_file_path)) + + ctx.logger.warn('Values file not found') + + +def _set_deployment_result(key): + result = ctx.instance.runtime_properties.pop(key) + ctx.instance.runtime_properties['kubernetes'] = result + + +def _do_delete_resources(kubernetes_plugin, yaml, jinja_env, values): + for path in _retrieve_resources_paths(): + ctx.logger.info('Deleting resource defined in: {0}'.format(path)) + + template_content = ctx.get_resource(path) + yaml_content = _render_template( + jinja_env, + template_content, + values + ) + content = yaml.load(yaml_content) + + _set_deployment_result( + 'resource_{0}'.format(content['metadata']['name']) + ) + kubernetes_plugin.resource_delete(definition=content) + + ctx.logger.info('Resources deleted successfully') + + +def _do_delete_services(kubernetes_plugin, yaml, jinja_env, values): + ctx.logger.info('Deleting services') + services_file_path = _retrieve_services_paths() + + if not services_file_path: + ctx.logger.warn( + 'Service file is not defined. Skipping services provisioning !' + ) + + return + + template_content = ctx.get_resource(services_file_path) + yaml_content = _render_template( + jinja_env, + template_content, + values + ) + + yaml_content_parts = \ + yaml_content.split(SERVICES_FILE_PARTS_SEPARATOR) + + for yaml_content_part in yaml_content_parts: + content = yaml.load(yaml_content_part) + + _set_deployment_result( + 'service_{0}'.format(content['metadata']['name']) + ) + kubernetes_plugin.resource_delete(definition=content) + + ctx.logger.info('Services deleted successfully') + + +if __name__ == '__main__': + yaml, kubernetes_plugin, jinja2 = _import_or_install() + jinja_env = _init_jinja(jinja2) + values = _retrieve_values(yaml) + + _do_delete_services(kubernetes_plugin, yaml, jinja_env, values) + _do_delete_resources(kubernetes_plugin, yaml, jinja_env, values) + + diff --git a/cloudify/scripts/onap/patch_definitions.py b/cloudify/scripts/onap/patch_definitions.py deleted file mode 100644 index d43e921593..0000000000 --- a/cloudify/scripts/onap/patch_definitions.py +++ /dev/null @@ -1 +0,0 @@ -from cloudify import ctx diff --git a/cloudify/scripts/onap/provision_definitions.py b/cloudify/scripts/onap/provision_definitions.py deleted file mode 100644 index d43e921593..0000000000 --- a/cloudify/scripts/onap/provision_definitions.py +++ /dev/null @@ -1 +0,0 @@ -from cloudify import ctx diff --git a/cloudify/scripts/onap/read_definitions.py b/cloudify/scripts/onap/read_definitions.py deleted file mode 100644 index d43e921593..0000000000 --- a/cloudify/scripts/onap/read_definitions.py +++ /dev/null @@ -1 +0,0 @@ -from cloudify import ctx diff --git a/cloudify/types/onap.yaml b/cloudify/types/onap.yaml index 20ef33f2f3..7e9b83425e 100644 --- a/cloudify/types/onap.yaml +++ b/cloudify/types/onap.yaml @@ -1,4 +1,33 @@ node_types: + cloudify.onap.kubernetes.Environment: + derived_from: cloudify.nodes.Root + properties: + namespace: + type: string + init_pod: + type: string + description: > + Path to init pod YAML file + options: + description: > + For compatibility with kubernetes plugin. + To be removed in the future. + default: {} + interfaces: + cloudify.interfaces.lifecycle: + create: + implementation: cloudify/scripts/onap/create_namespace.py + executor: central_deployment_agent + start: + implementation: cloudify/scripts/onap/create_init_pod.py + executor: central_deployment_agent + stop: + implementation: cloudify/scripts/onap/delete_init_pod.py + executor: central_deployment_agent + delete: + implementation: cloudify/scripts/onap/delete_namespace.py + executor: central_deployment_agent + cloudify.onap.kubernetes.App: derived_from: cloudify.nodes.Root properties: @@ -6,6 +35,11 @@ node_types: type: string description: > Name of ONAP app + values: + type: string + description: > + Paths (relative, blueprint prespective) to values.yaml file + required: false resources: description: > List of paths (relative, blueprint prespective) @@ -21,14 +55,35 @@ node_types: description: > Parameters required to create kubernetes resources for each app default: {} + options: + description: > + For compatibility with kubernetes plugin. + To be removed in the future. + default: {} interfaces: cloudify.interfaces.lifecycle: create: - implementation: cloudify/scripts/onap/read_definitions.py + implementation: cloudify/scripts/onap/create_namespace.py executor: central_deployment_agent configure: - implementation: cloudify/scripts/onap/patch_definitions.py + implementation: fabric.fabric_plugin.tasks.run_task executor: central_deployment_agent + inputs: + tasks_file: + default: cloudify/scripts/onap/configure_docker_secret_workaround.py + task_name: + default: configure_secret + fabric_env: + default: + host_string: { get_secret: kubernetes_master_ip } + user: { get_secret: agent_user } + key: { get_secret: agent_key_private } start: - implementation: cloudify/scripts/onap/provision_definitions.py + implementation: cloudify/scripts/onap/create_resources_services.py + executor: central_deployment_agent + stop: + implementation: cloudify/scripts/onap/delete_resources_services.py + executor: central_deployment_agent + delete: + implementation: cloudify/scripts/onap/delete_namespace.py executor: central_deployment_agent -- cgit 1.2.3-korg