diff options
Diffstat (limited to 'aria/multivim-plugin/system_tests/openstack_handler.py')
| -rw-r--r-- | aria/multivim-plugin/system_tests/openstack_handler.py | 657 |
1 files changed, 657 insertions, 0 deletions
diff --git a/aria/multivim-plugin/system_tests/openstack_handler.py b/aria/multivim-plugin/system_tests/openstack_handler.py new file mode 100644 index 0000000000..76368fa10a --- /dev/null +++ b/aria/multivim-plugin/system_tests/openstack_handler.py @@ -0,0 +1,657 @@ +######## +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +import random +import logging +import os +import time +import copy +from contextlib import contextmanager + +from cinderclient import client as cinderclient +from keystoneauth1 import loading, session +import novaclient.client as nvclient +import neutronclient.v2_0.client as neclient +from retrying import retry + +from cosmo_tester.framework.handlers import ( + BaseHandler, + BaseCloudifyInputsConfigReader) +from cosmo_tester.framework.util import get_actual_keypath + +logging.getLogger('neutronclient.client').setLevel(logging.INFO) +logging.getLogger('novaclient.client').setLevel(logging.INFO) + + +VOLUME_TERMINATION_TIMEOUT_SECS = 300 + + +class OpenstackCleanupContext(BaseHandler.CleanupContext): + + def __init__(self, context_name, env): + super(OpenstackCleanupContext, self).__init__(context_name, env) + self.before_run = self.env.handler.openstack_infra_state() + + def cleanup(self): + """ + Cleans resources created by the test. + Resource that existed before the test will not be removed + """ + super(OpenstackCleanupContext, self).cleanup() + resources_to_teardown = self.get_resources_to_teardown( + self.env, resources_to_keep=self.before_run) + if self.skip_cleanup: + self.logger.warn('[{0}] SKIPPING cleanup of resources: {1}' + .format(self.context_name, resources_to_teardown)) + else: + self._clean(self.env, resources_to_teardown) + + @classmethod + def clean_all(cls, env): + """ + Cleans *all* resources, including resources that were not + created by the test + """ + super(OpenstackCleanupContext, cls).clean_all(env) + resources_to_teardown = cls.get_resources_to_teardown(env) + cls._clean(env, resources_to_teardown) + + @classmethod + def _clean(cls, env, resources_to_teardown): + cls.logger.info('Openstack handler will try to remove these resources:' + ' {0}'.format(resources_to_teardown)) + failed_to_remove = env.handler.remove_openstack_resources( + resources_to_teardown) + if failed_to_remove: + trimmed_failed_to_remove = {key: value for key, value in + failed_to_remove.iteritems() + if value} + if len(trimmed_failed_to_remove) > 0: + msg = 'Openstack handler failed to remove some resources:' \ + ' {0}'.format(trimmed_failed_to_remove) + cls.logger.error(msg) + raise RuntimeError(msg) + + @classmethod + def get_resources_to_teardown(cls, env, resources_to_keep=None): + all_existing_resources = env.handler.openstack_infra_state() + if resources_to_keep: + return env.handler.openstack_infra_state_delta( + before=resources_to_keep, after=all_existing_resources) + else: + return all_existing_resources + + def update_server_id(self, server_name): + + # retrieve the id of the new server + nova, _, _ = self.env.handler.openstack_clients() + servers = nova.servers.list( + search_opts={'name': server_name}) + if len(servers) > 1: + raise RuntimeError( + 'Expected 1 server with name {0}, but found {1}' + .format(server_name, len(servers))) + + new_server_id = servers[0].id + + # retrieve the id of the old server + old_server_id = None + servers = self.before_run['servers'] + for server_id, name in servers.iteritems(): + if server_name == name: + old_server_id = server_id + break + if old_server_id is None: + raise RuntimeError( + 'Could not find a server with name {0} ' + 'in the internal cleanup context state' + .format(server_name)) + + # replace the id in the internal state + servers[new_server_id] = servers.pop(old_server_id) + + +class CloudifyOpenstackInputsConfigReader(BaseCloudifyInputsConfigReader): + + def __init__(self, cloudify_config, manager_blueprint_path, **kwargs): + super(CloudifyOpenstackInputsConfigReader, self).__init__( + cloudify_config, manager_blueprint_path=manager_blueprint_path, + **kwargs) + + @property + def region(self): + return self.config['region'] + + @property + def management_server_name(self): + return self.config['manager_server_name'] + + @property + def agent_key_path(self): + return self.config['agent_private_key_path'] + + @property + def management_user_name(self): + return self.config['ssh_user'] + + @property + def management_key_path(self): + return self.config['ssh_key_filename'] + + @property + def agent_keypair_name(self): + return self.config['agent_public_key_name'] + + @property + def management_keypair_name(self): + return self.config['manager_public_key_name'] + + @property + def use_existing_agent_keypair(self): + return self.config['use_existing_agent_keypair'] + + @property + def use_existing_manager_keypair(self): + return self.config['use_existing_manager_keypair'] + + @property + def external_network_name(self): + return self.config['external_network_name'] + + @property + def keystone_username(self): + return self.config['keystone_username'] + + @property + def keystone_password(self): + return self.config['keystone_password'] + + @property + def keystone_tenant_name(self): + return self.config['keystone_tenant_name'] + + @property + def keystone_url(self): + return self.config['keystone_url'] + + @property + def neutron_url(self): + return self.config.get('neutron_url', None) + + @property + def management_network_name(self): + return self.config['management_network_name'] + + @property + def management_subnet_name(self): + return self.config['management_subnet_name'] + + @property + def management_router_name(self): + return self.config['management_router'] + + @property + def agents_security_group(self): + return self.config['agents_security_group_name'] + + @property + def management_security_group(self): + return self.config['manager_security_group_name'] + + +class OpenstackHandler(BaseHandler): + + CleanupContext = OpenstackCleanupContext + CloudifyConfigReader = CloudifyOpenstackInputsConfigReader + + def before_bootstrap(self): + super(OpenstackHandler, self).before_bootstrap() + with self.update_cloudify_config() as patch: + suffix = '-%06x' % random.randrange(16 ** 6) + server_name_prop_path = 'manager_server_name' + patch.append_value(server_name_prop_path, suffix) + + def after_bootstrap(self, provider_context): + super(OpenstackHandler, self).after_bootstrap(provider_context) + resources = provider_context['resources'] + agent_keypair = resources['agents_keypair'] + management_keypair = resources['management_keypair'] + self.remove_agent_keypair = agent_keypair['external_resource'] is False + self.remove_management_keypair = \ + management_keypair['external_resource'] is False + + def after_teardown(self): + super(OpenstackHandler, self).after_teardown() + if self.remove_agent_keypair: + agent_key_path = get_actual_keypath(self.env, + self.env.agent_key_path, + raise_on_missing=False) + if agent_key_path: + os.remove(agent_key_path) + if self.remove_management_keypair: + management_key_path = get_actual_keypath( + self.env, + self.env.management_key_path, + raise_on_missing=False) + if management_key_path: + os.remove(management_key_path) + + def openstack_clients(self): + creds = self._client_creds() + params = { + 'region_name': creds.pop('region_name'), + } + + loader = loading.get_plugin_loader("password") + auth = loader.load_from_options(**creds) + sess = session.Session(auth=auth, verify=True) + + params['session'] = sess + + nova = nvclient.Client('2', **params) + neutron = neclient.Client(**params) + cinder = cinderclient.Client('2', **params) + + return (nova, neutron, cinder) + + @retry(stop_max_attempt_number=5, wait_fixed=20000) + def openstack_infra_state(self): + """ + @retry decorator is used because this error sometimes occur: + ConnectionFailed: Connection to neutron failed: Maximum + attempts reached + """ + nova, neutron, cinder = self.openstack_clients() + try: + prefix = self.env.resources_prefix + except (AttributeError, KeyError): + prefix = '' + return { + 'networks': dict(self._networks(neutron, prefix)), + 'subnets': dict(self._subnets(neutron, prefix)), + 'routers': dict(self._routers(neutron, prefix)), + 'security_groups': dict(self._security_groups(neutron, prefix)), + 'servers': dict(self._servers(nova, prefix)), + 'key_pairs': dict(self._key_pairs(nova, prefix)), + 'floatingips': dict(self._floatingips(neutron, prefix)), + 'ports': dict(self._ports(neutron, prefix)), + 'volumes': dict(self._volumes(cinder, prefix)) + } + + def openstack_infra_state_delta(self, before, after): + after = copy.deepcopy(after) + return { + prop: self._remove_keys(after[prop], before[prop].keys()) + for prop in before + } + + def _find_keypairs_to_delete(self, nodes, node_instances): + """Filter the nodes only returning the names of keypair nodes + + Examine node_instances and nodes, return the external_name of + those node_instances, which correspond to a node that has a + type == KeyPair + + To filter by deployment_id, simply make sure that the nodes and + node_instances this method receives, are pre-filtered + (ie. filter the nodes while fetching them from the manager) + """ + keypairs = set() # a set of (deployment_id, node_id) tuples + + for node in nodes: + if node.get('type') != 'cloudify.openstack.nodes.KeyPair': + continue + # deployment_id isnt always present in local_env runs + key = (node.get('deployment_id'), node['id']) + keypairs.add(key) + + for node_instance in node_instances: + key = (node_instance.get('deployment_id'), + node_instance['node_id']) + if key not in keypairs: + continue + + runtime_properties = node_instance['runtime_properties'] + if not runtime_properties: + continue + name = runtime_properties.get('external_name') + if name: + yield name + + def _delete_keypairs_by_name(self, keypair_names): + nova, neutron, cinder = self.openstack_clients() + existing_keypairs = nova.keypairs.list() + + for name in keypair_names: + for keypair in existing_keypairs: + if keypair.name == name: + nova.keypairs.delete(keypair) + + def remove_keypairs_from_local_env(self, local_env): + """Query the local_env for nodes which are keypairs, remove them + + Similar to querying the manager, we can look up nodes in the local_env + which is used for tests. + """ + nodes = local_env.storage.get_nodes() + node_instances = local_env.storage.get_node_instances() + names = self._find_keypairs_to_delete(nodes, node_instances) + self._delete_keypairs_by_name(names) + + def remove_keypairs_from_manager(self, deployment_id=None, + rest_client=None): + """Query the manager for nodes by deployment_id, delete keypairs + + Fetch nodes and node_instances from the manager by deployment_id + (or all if not given), find which ones represent openstack keypairs, + remove them. + """ + if rest_client is None: + rest_client = self.env.rest_client + + nodes = rest_client.nodes.list(deployment_id=deployment_id) + node_instances = rest_client.node_instances.list( + deployment_id=deployment_id) + keypairs = self._find_keypairs_to_delete(nodes, node_instances) + self._delete_keypairs_by_name(keypairs) + + def remove_keypair(self, name): + """Delete an openstack keypair by name. If it doesnt exist, do nothing. + """ + self._delete_keypairs_by_name([name]) + + def remove_openstack_resources(self, resources_to_remove): + # basically sort of a workaround, but if we get the order wrong + # the first time, there is a chance things would better next time + # 3'rd time can't really hurt, can it? + # 3 is a charm + for _ in range(3): + resources_to_remove = self._remove_openstack_resources_impl( + resources_to_remove) + if all([len(g) == 0 for g in resources_to_remove.values()]): + break + # give openstack some time to update its data structures + time.sleep(3) + return resources_to_remove + + def _remove_openstack_resources_impl(self, resources_to_remove): + nova, neutron, cinder = self.openstack_clients() + + servers = nova.servers.list() + ports = neutron.list_ports()['ports'] + routers = neutron.list_routers()['routers'] + subnets = neutron.list_subnets()['subnets'] + networks = neutron.list_networks()['networks'] + # keypairs = nova.keypairs.list() + floatingips = neutron.list_floatingips()['floatingips'] + security_groups = neutron.list_security_groups()['security_groups'] + volumes = cinder.volumes.list() + + failed = { + 'servers': {}, + 'routers': {}, + 'ports': {}, + 'subnets': {}, + 'networks': {}, + 'key_pairs': {}, + 'floatingips': {}, + 'security_groups': {}, + 'volumes': {} + } + + volumes_to_remove = [] + for volume in volumes: + if volume.id in resources_to_remove['volumes']: + volumes_to_remove.append(volume) + + left_volumes = self._delete_volumes(nova, cinder, volumes_to_remove) + for volume_id, ex in left_volumes.iteritems(): + failed['volumes'][volume_id] = ex + + for server in servers: + if server.id in resources_to_remove['servers']: + with self._handled_exception(server.id, failed, 'servers'): + nova.servers.delete(server) + + for router in routers: + if router['id'] in resources_to_remove['routers']: + with self._handled_exception(router['id'], failed, 'routers'): + for p in neutron.list_ports( + device_id=router['id'])['ports']: + neutron.remove_interface_router(router['id'], { + 'port_id': p['id'] + }) + neutron.delete_router(router['id']) + + for port in ports: + if port['id'] in resources_to_remove['ports']: + with self._handled_exception(port['id'], failed, 'ports'): + neutron.delete_port(port['id']) + + for subnet in subnets: + if subnet['id'] in resources_to_remove['subnets']: + with self._handled_exception(subnet['id'], failed, 'subnets'): + neutron.delete_subnet(subnet['id']) + + for network in networks: + if network['name'] == self.env.external_network_name: + continue + if network['id'] in resources_to_remove['networks']: + with self._handled_exception(network['id'], failed, + 'networks'): + neutron.delete_network(network['id']) + + # TODO: implement key-pair creation and cleanup per tenant + # + # IMPORTANT: Do not remove key-pairs, they might be used + # by another tenant (of the same user) + # + # for key_pair in keypairs: + # if key_pair.name == self.env.agent_keypair_name and \ + # self.env.use_existing_agent_keypair: + # # this is a pre-existing agent key-pair, do not remove + # continue + # elif key_pair.name == self.env.management_keypair_name and \ + # self.env.use_existing_manager_keypair: + # # this is a pre-existing manager key-pair, do not remove + # continue + # elif key_pair.id in resources_to_remove['key_pairs']: + # with self._handled_exception(key_pair.id, failed, + # 'key_pairs'): + # nova.keypairs.delete(key_pair) + + for floatingip in floatingips: + if floatingip['id'] in resources_to_remove['floatingips']: + with self._handled_exception(floatingip['id'], failed, + 'floatingips'): + neutron.delete_floatingip(floatingip['id']) + + for security_group in security_groups: + if security_group['name'] == 'default': + continue + if security_group['id'] in resources_to_remove['security_groups']: + with self._handled_exception(security_group['id'], + failed, 'security_groups'): + neutron.delete_security_group(security_group['id']) + + return failed + + def _delete_volumes(self, nova, cinder, existing_volumes): + unremovables = {} + end_time = time.time() + VOLUME_TERMINATION_TIMEOUT_SECS + + for volume in existing_volumes: + # detach the volume + if volume.status in ['available', 'error', 'in-use']: + try: + self.logger.info('Detaching volume {0} ({1}), currently in' + ' status {2} ...'. + format(volume.name, volume.id, + volume.status)) + for attachment in volume.attachments: + nova.volumes.delete_server_volume( + server_id=attachment['server_id'], + attachment_id=attachment['id']) + except Exception as e: + self.logger.warning('Attempt to detach volume {0} ({1})' + ' yielded exception: "{2}"'. + format(volume.name, volume.id, + e)) + unremovables[volume.id] = e + existing_volumes.remove(volume) + + time.sleep(3) + for volume in existing_volumes: + # delete the volume + if volume.status in ['available', 'error', 'in-use']: + try: + self.logger.info('Deleting volume {0} ({1}), currently in' + ' status {2} ...'. + format(volume.name, volume.id, + volume.status)) + cinder.volumes.delete(volume) + except Exception as e: + self.logger.warning('Attempt to delete volume {0} ({1})' + ' yielded exception: "{2}"'. + format(volume.name, volume.id, + e)) + unremovables[volume.id] = e + existing_volumes.remove(volume) + + # wait for all volumes deletion until completed or timeout is reached + while existing_volumes and time.time() < end_time: + time.sleep(3) + for volume in existing_volumes: + volume_id = volume.id + volume_name = volume.name + try: + vol = cinder.volumes.get(volume_id) + if vol.status == 'deleting': + self.logger.debug('volume {0} ({1}) is being ' + 'deleted...'.format(volume_name, + volume_id)) + else: + self.logger.warning('volume {0} ({1}) is in ' + 'unexpected status: {2}'. + format(volume_name, volume_id, + vol.status)) + except Exception as e: + # the volume wasn't found, it was deleted + if hasattr(e, 'code') and e.code == 404: + self.logger.info('deleted volume {0} ({1})'. + format(volume_name, volume_id)) + existing_volumes.remove(volume) + else: + self.logger.warning('failed to remove volume {0} ' + '({1}), exception: {2}'. + format(volume_name, + volume_id, e)) + unremovables[volume_id] = e + existing_volumes.remove(volume) + + if existing_volumes: + for volume in existing_volumes: + # try to get the volume's status + try: + vol = cinder.volumes.get(volume.id) + vol_status = vol.status + except: + # failed to get volume... status is unknown + vol_status = 'unknown' + + unremovables[volume.id] = 'timed out while removing volume '\ + '{0} ({1}), current volume status '\ + 'is {2}'.format(volume.name, + volume.id, + vol_status) + + if unremovables: + self.logger.warning('failed to remove volumes: {0}'.format( + unremovables)) + + return unremovables + + def _client_creds(self): + return { + 'username': self.env.keystone_username, + 'password': self.env.keystone_password, + 'auth_url': self.env.keystone_url, + 'project_name': self.env.keystone_tenant_name, + 'region_name': self.env.region + } + + def _networks(self, neutron, prefix): + return [(n['id'], n['name']) + for n in neutron.list_networks()['networks'] + if self._check_prefix(n['name'], prefix)] + + def _subnets(self, neutron, prefix): + return [(n['id'], n['name']) + for n in neutron.list_subnets()['subnets'] + if self._check_prefix(n['name'], prefix)] + + def _routers(self, neutron, prefix): + return [(n['id'], n['name']) + for n in neutron.list_routers()['routers'] + if self._check_prefix(n['name'], prefix)] + + def _security_groups(self, neutron, prefix): + return [(n['id'], n['name']) + for n in neutron.list_security_groups()['security_groups'] + if self._check_prefix(n['name'], prefix)] + + def _servers(self, nova, prefix): + return [(s.id, s.human_id) + for s in nova.servers.list() + if self._check_prefix(s.human_id, prefix)] + + def _key_pairs(self, nova, prefix): + return [(kp.id, kp.name) + for kp in nova.keypairs.list() + if self._check_prefix(kp.name, prefix)] + + def _floatingips(self, neutron, prefix): + return [(ip['id'], ip['floating_ip_address']) + for ip in neutron.list_floatingips()['floatingips']] + + def _ports(self, neutron, prefix): + return [(p['id'], p['name']) + for p in neutron.list_ports()['ports'] + if self._check_prefix(p['name'], prefix)] + + def _volumes(self, cinder, prefix): + return [(v.id, v.name) for v in cinder.volumes.list() + if self._check_prefix(v.name, prefix)] + + def _check_prefix(self, name, prefix): + # some openstack resources (eg. volumes) can have no display_name, + # in which case it's None + return name is None or name.startswith(prefix) + + def _remove_keys(self, dct, keys): + for key in keys: + if key in dct: + del dct[key] + return dct + + @contextmanager + def _handled_exception(self, resource_id, failed, resource_group): + try: + yield + except BaseException, ex: + failed[resource_group][resource_id] = ex + + +handler = OpenstackHandler |