summaryrefslogtreecommitdiffstats
path: root/aria/multivim-plugin/system_tests/openstack_handler.py
diff options
context:
space:
mode:
Diffstat (limited to 'aria/multivim-plugin/system_tests/openstack_handler.py')
-rw-r--r--aria/multivim-plugin/system_tests/openstack_handler.py657
1 files changed, 657 insertions, 0 deletions
diff --git a/aria/multivim-plugin/system_tests/openstack_handler.py b/aria/multivim-plugin/system_tests/openstack_handler.py
new file mode 100644
index 0000000000..76368fa10a
--- /dev/null
+++ b/aria/multivim-plugin/system_tests/openstack_handler.py
@@ -0,0 +1,657 @@
+########
+# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+
+import random
+import logging
+import os
+import time
+import copy
+from contextlib import contextmanager
+
+from cinderclient import client as cinderclient
+from keystoneauth1 import loading, session
+import novaclient.client as nvclient
+import neutronclient.v2_0.client as neclient
+from retrying import retry
+
+from cosmo_tester.framework.handlers import (
+ BaseHandler,
+ BaseCloudifyInputsConfigReader)
+from cosmo_tester.framework.util import get_actual_keypath
+
+logging.getLogger('neutronclient.client').setLevel(logging.INFO)
+logging.getLogger('novaclient.client').setLevel(logging.INFO)
+
+
+VOLUME_TERMINATION_TIMEOUT_SECS = 300
+
+
+class OpenstackCleanupContext(BaseHandler.CleanupContext):
+
+ def __init__(self, context_name, env):
+ super(OpenstackCleanupContext, self).__init__(context_name, env)
+ self.before_run = self.env.handler.openstack_infra_state()
+
+ def cleanup(self):
+ """
+ Cleans resources created by the test.
+ Resource that existed before the test will not be removed
+ """
+ super(OpenstackCleanupContext, self).cleanup()
+ resources_to_teardown = self.get_resources_to_teardown(
+ self.env, resources_to_keep=self.before_run)
+ if self.skip_cleanup:
+ self.logger.warn('[{0}] SKIPPING cleanup of resources: {1}'
+ .format(self.context_name, resources_to_teardown))
+ else:
+ self._clean(self.env, resources_to_teardown)
+
+ @classmethod
+ def clean_all(cls, env):
+ """
+ Cleans *all* resources, including resources that were not
+ created by the test
+ """
+ super(OpenstackCleanupContext, cls).clean_all(env)
+ resources_to_teardown = cls.get_resources_to_teardown(env)
+ cls._clean(env, resources_to_teardown)
+
+ @classmethod
+ def _clean(cls, env, resources_to_teardown):
+ cls.logger.info('Openstack handler will try to remove these resources:'
+ ' {0}'.format(resources_to_teardown))
+ failed_to_remove = env.handler.remove_openstack_resources(
+ resources_to_teardown)
+ if failed_to_remove:
+ trimmed_failed_to_remove = {key: value for key, value in
+ failed_to_remove.iteritems()
+ if value}
+ if len(trimmed_failed_to_remove) > 0:
+ msg = 'Openstack handler failed to remove some resources:' \
+ ' {0}'.format(trimmed_failed_to_remove)
+ cls.logger.error(msg)
+ raise RuntimeError(msg)
+
+ @classmethod
+ def get_resources_to_teardown(cls, env, resources_to_keep=None):
+ all_existing_resources = env.handler.openstack_infra_state()
+ if resources_to_keep:
+ return env.handler.openstack_infra_state_delta(
+ before=resources_to_keep, after=all_existing_resources)
+ else:
+ return all_existing_resources
+
+ def update_server_id(self, server_name):
+
+ # retrieve the id of the new server
+ nova, _, _ = self.env.handler.openstack_clients()
+ servers = nova.servers.list(
+ search_opts={'name': server_name})
+ if len(servers) > 1:
+ raise RuntimeError(
+ 'Expected 1 server with name {0}, but found {1}'
+ .format(server_name, len(servers)))
+
+ new_server_id = servers[0].id
+
+ # retrieve the id of the old server
+ old_server_id = None
+ servers = self.before_run['servers']
+ for server_id, name in servers.iteritems():
+ if server_name == name:
+ old_server_id = server_id
+ break
+ if old_server_id is None:
+ raise RuntimeError(
+ 'Could not find a server with name {0} '
+ 'in the internal cleanup context state'
+ .format(server_name))
+
+ # replace the id in the internal state
+ servers[new_server_id] = servers.pop(old_server_id)
+
+
+class CloudifyOpenstackInputsConfigReader(BaseCloudifyInputsConfigReader):
+
+ def __init__(self, cloudify_config, manager_blueprint_path, **kwargs):
+ super(CloudifyOpenstackInputsConfigReader, self).__init__(
+ cloudify_config, manager_blueprint_path=manager_blueprint_path,
+ **kwargs)
+
+ @property
+ def region(self):
+ return self.config['region']
+
+ @property
+ def management_server_name(self):
+ return self.config['manager_server_name']
+
+ @property
+ def agent_key_path(self):
+ return self.config['agent_private_key_path']
+
+ @property
+ def management_user_name(self):
+ return self.config['ssh_user']
+
+ @property
+ def management_key_path(self):
+ return self.config['ssh_key_filename']
+
+ @property
+ def agent_keypair_name(self):
+ return self.config['agent_public_key_name']
+
+ @property
+ def management_keypair_name(self):
+ return self.config['manager_public_key_name']
+
+ @property
+ def use_existing_agent_keypair(self):
+ return self.config['use_existing_agent_keypair']
+
+ @property
+ def use_existing_manager_keypair(self):
+ return self.config['use_existing_manager_keypair']
+
+ @property
+ def external_network_name(self):
+ return self.config['external_network_name']
+
+ @property
+ def keystone_username(self):
+ return self.config['keystone_username']
+
+ @property
+ def keystone_password(self):
+ return self.config['keystone_password']
+
+ @property
+ def keystone_tenant_name(self):
+ return self.config['keystone_tenant_name']
+
+ @property
+ def keystone_url(self):
+ return self.config['keystone_url']
+
+ @property
+ def neutron_url(self):
+ return self.config.get('neutron_url', None)
+
+ @property
+ def management_network_name(self):
+ return self.config['management_network_name']
+
+ @property
+ def management_subnet_name(self):
+ return self.config['management_subnet_name']
+
+ @property
+ def management_router_name(self):
+ return self.config['management_router']
+
+ @property
+ def agents_security_group(self):
+ return self.config['agents_security_group_name']
+
+ @property
+ def management_security_group(self):
+ return self.config['manager_security_group_name']
+
+
+class OpenstackHandler(BaseHandler):
+
+ CleanupContext = OpenstackCleanupContext
+ CloudifyConfigReader = CloudifyOpenstackInputsConfigReader
+
+ def before_bootstrap(self):
+ super(OpenstackHandler, self).before_bootstrap()
+ with self.update_cloudify_config() as patch:
+ suffix = '-%06x' % random.randrange(16 ** 6)
+ server_name_prop_path = 'manager_server_name'
+ patch.append_value(server_name_prop_path, suffix)
+
+ def after_bootstrap(self, provider_context):
+ super(OpenstackHandler, self).after_bootstrap(provider_context)
+ resources = provider_context['resources']
+ agent_keypair = resources['agents_keypair']
+ management_keypair = resources['management_keypair']
+ self.remove_agent_keypair = agent_keypair['external_resource'] is False
+ self.remove_management_keypair = \
+ management_keypair['external_resource'] is False
+
+ def after_teardown(self):
+ super(OpenstackHandler, self).after_teardown()
+ if self.remove_agent_keypair:
+ agent_key_path = get_actual_keypath(self.env,
+ self.env.agent_key_path,
+ raise_on_missing=False)
+ if agent_key_path:
+ os.remove(agent_key_path)
+ if self.remove_management_keypair:
+ management_key_path = get_actual_keypath(
+ self.env,
+ self.env.management_key_path,
+ raise_on_missing=False)
+ if management_key_path:
+ os.remove(management_key_path)
+
+ def openstack_clients(self):
+ creds = self._client_creds()
+ params = {
+ 'region_name': creds.pop('region_name'),
+ }
+
+ loader = loading.get_plugin_loader("password")
+ auth = loader.load_from_options(**creds)
+ sess = session.Session(auth=auth, verify=True)
+
+ params['session'] = sess
+
+ nova = nvclient.Client('2', **params)
+ neutron = neclient.Client(**params)
+ cinder = cinderclient.Client('2', **params)
+
+ return (nova, neutron, cinder)
+
+ @retry(stop_max_attempt_number=5, wait_fixed=20000)
+ def openstack_infra_state(self):
+ """
+ @retry decorator is used because this error sometimes occur:
+ ConnectionFailed: Connection to neutron failed: Maximum
+ attempts reached
+ """
+ nova, neutron, cinder = self.openstack_clients()
+ try:
+ prefix = self.env.resources_prefix
+ except (AttributeError, KeyError):
+ prefix = ''
+ return {
+ 'networks': dict(self._networks(neutron, prefix)),
+ 'subnets': dict(self._subnets(neutron, prefix)),
+ 'routers': dict(self._routers(neutron, prefix)),
+ 'security_groups': dict(self._security_groups(neutron, prefix)),
+ 'servers': dict(self._servers(nova, prefix)),
+ 'key_pairs': dict(self._key_pairs(nova, prefix)),
+ 'floatingips': dict(self._floatingips(neutron, prefix)),
+ 'ports': dict(self._ports(neutron, prefix)),
+ 'volumes': dict(self._volumes(cinder, prefix))
+ }
+
+ def openstack_infra_state_delta(self, before, after):
+ after = copy.deepcopy(after)
+ return {
+ prop: self._remove_keys(after[prop], before[prop].keys())
+ for prop in before
+ }
+
+ def _find_keypairs_to_delete(self, nodes, node_instances):
+ """Filter the nodes only returning the names of keypair nodes
+
+ Examine node_instances and nodes, return the external_name of
+ those node_instances, which correspond to a node that has a
+ type == KeyPair
+
+ To filter by deployment_id, simply make sure that the nodes and
+ node_instances this method receives, are pre-filtered
+ (ie. filter the nodes while fetching them from the manager)
+ """
+ keypairs = set() # a set of (deployment_id, node_id) tuples
+
+ for node in nodes:
+ if node.get('type') != 'cloudify.openstack.nodes.KeyPair':
+ continue
+ # deployment_id isnt always present in local_env runs
+ key = (node.get('deployment_id'), node['id'])
+ keypairs.add(key)
+
+ for node_instance in node_instances:
+ key = (node_instance.get('deployment_id'),
+ node_instance['node_id'])
+ if key not in keypairs:
+ continue
+
+ runtime_properties = node_instance['runtime_properties']
+ if not runtime_properties:
+ continue
+ name = runtime_properties.get('external_name')
+ if name:
+ yield name
+
+ def _delete_keypairs_by_name(self, keypair_names):
+ nova, neutron, cinder = self.openstack_clients()
+ existing_keypairs = nova.keypairs.list()
+
+ for name in keypair_names:
+ for keypair in existing_keypairs:
+ if keypair.name == name:
+ nova.keypairs.delete(keypair)
+
+ def remove_keypairs_from_local_env(self, local_env):
+ """Query the local_env for nodes which are keypairs, remove them
+
+ Similar to querying the manager, we can look up nodes in the local_env
+ which is used for tests.
+ """
+ nodes = local_env.storage.get_nodes()
+ node_instances = local_env.storage.get_node_instances()
+ names = self._find_keypairs_to_delete(nodes, node_instances)
+ self._delete_keypairs_by_name(names)
+
+ def remove_keypairs_from_manager(self, deployment_id=None,
+ rest_client=None):
+ """Query the manager for nodes by deployment_id, delete keypairs
+
+ Fetch nodes and node_instances from the manager by deployment_id
+ (or all if not given), find which ones represent openstack keypairs,
+ remove them.
+ """
+ if rest_client is None:
+ rest_client = self.env.rest_client
+
+ nodes = rest_client.nodes.list(deployment_id=deployment_id)
+ node_instances = rest_client.node_instances.list(
+ deployment_id=deployment_id)
+ keypairs = self._find_keypairs_to_delete(nodes, node_instances)
+ self._delete_keypairs_by_name(keypairs)
+
+ def remove_keypair(self, name):
+ """Delete an openstack keypair by name. If it doesnt exist, do nothing.
+ """
+ self._delete_keypairs_by_name([name])
+
+ def remove_openstack_resources(self, resources_to_remove):
+ # basically sort of a workaround, but if we get the order wrong
+ # the first time, there is a chance things would better next time
+ # 3'rd time can't really hurt, can it?
+ # 3 is a charm
+ for _ in range(3):
+ resources_to_remove = self._remove_openstack_resources_impl(
+ resources_to_remove)
+ if all([len(g) == 0 for g in resources_to_remove.values()]):
+ break
+ # give openstack some time to update its data structures
+ time.sleep(3)
+ return resources_to_remove
+
+ def _remove_openstack_resources_impl(self, resources_to_remove):
+ nova, neutron, cinder = self.openstack_clients()
+
+ servers = nova.servers.list()
+ ports = neutron.list_ports()['ports']
+ routers = neutron.list_routers()['routers']
+ subnets = neutron.list_subnets()['subnets']
+ networks = neutron.list_networks()['networks']
+ # keypairs = nova.keypairs.list()
+ floatingips = neutron.list_floatingips()['floatingips']
+ security_groups = neutron.list_security_groups()['security_groups']
+ volumes = cinder.volumes.list()
+
+ failed = {
+ 'servers': {},
+ 'routers': {},
+ 'ports': {},
+ 'subnets': {},
+ 'networks': {},
+ 'key_pairs': {},
+ 'floatingips': {},
+ 'security_groups': {},
+ 'volumes': {}
+ }
+
+ volumes_to_remove = []
+ for volume in volumes:
+ if volume.id in resources_to_remove['volumes']:
+ volumes_to_remove.append(volume)
+
+ left_volumes = self._delete_volumes(nova, cinder, volumes_to_remove)
+ for volume_id, ex in left_volumes.iteritems():
+ failed['volumes'][volume_id] = ex
+
+ for server in servers:
+ if server.id in resources_to_remove['servers']:
+ with self._handled_exception(server.id, failed, 'servers'):
+ nova.servers.delete(server)
+
+ for router in routers:
+ if router['id'] in resources_to_remove['routers']:
+ with self._handled_exception(router['id'], failed, 'routers'):
+ for p in neutron.list_ports(
+ device_id=router['id'])['ports']:
+ neutron.remove_interface_router(router['id'], {
+ 'port_id': p['id']
+ })
+ neutron.delete_router(router['id'])
+
+ for port in ports:
+ if port['id'] in resources_to_remove['ports']:
+ with self._handled_exception(port['id'], failed, 'ports'):
+ neutron.delete_port(port['id'])
+
+ for subnet in subnets:
+ if subnet['id'] in resources_to_remove['subnets']:
+ with self._handled_exception(subnet['id'], failed, 'subnets'):
+ neutron.delete_subnet(subnet['id'])
+
+ for network in networks:
+ if network['name'] == self.env.external_network_name:
+ continue
+ if network['id'] in resources_to_remove['networks']:
+ with self._handled_exception(network['id'], failed,
+ 'networks'):
+ neutron.delete_network(network['id'])
+
+ # TODO: implement key-pair creation and cleanup per tenant
+ #
+ # IMPORTANT: Do not remove key-pairs, they might be used
+ # by another tenant (of the same user)
+ #
+ # for key_pair in keypairs:
+ # if key_pair.name == self.env.agent_keypair_name and \
+ # self.env.use_existing_agent_keypair:
+ # # this is a pre-existing agent key-pair, do not remove
+ # continue
+ # elif key_pair.name == self.env.management_keypair_name and \
+ # self.env.use_existing_manager_keypair:
+ # # this is a pre-existing manager key-pair, do not remove
+ # continue
+ # elif key_pair.id in resources_to_remove['key_pairs']:
+ # with self._handled_exception(key_pair.id, failed,
+ # 'key_pairs'):
+ # nova.keypairs.delete(key_pair)
+
+ for floatingip in floatingips:
+ if floatingip['id'] in resources_to_remove['floatingips']:
+ with self._handled_exception(floatingip['id'], failed,
+ 'floatingips'):
+ neutron.delete_floatingip(floatingip['id'])
+
+ for security_group in security_groups:
+ if security_group['name'] == 'default':
+ continue
+ if security_group['id'] in resources_to_remove['security_groups']:
+ with self._handled_exception(security_group['id'],
+ failed, 'security_groups'):
+ neutron.delete_security_group(security_group['id'])
+
+ return failed
+
+ def _delete_volumes(self, nova, cinder, existing_volumes):
+ unremovables = {}
+ end_time = time.time() + VOLUME_TERMINATION_TIMEOUT_SECS
+
+ for volume in existing_volumes:
+ # detach the volume
+ if volume.status in ['available', 'error', 'in-use']:
+ try:
+ self.logger.info('Detaching volume {0} ({1}), currently in'
+ ' status {2} ...'.
+ format(volume.name, volume.id,
+ volume.status))
+ for attachment in volume.attachments:
+ nova.volumes.delete_server_volume(
+ server_id=attachment['server_id'],
+ attachment_id=attachment['id'])
+ except Exception as e:
+ self.logger.warning('Attempt to detach volume {0} ({1})'
+ ' yielded exception: "{2}"'.
+ format(volume.name, volume.id,
+ e))
+ unremovables[volume.id] = e
+ existing_volumes.remove(volume)
+
+ time.sleep(3)
+ for volume in existing_volumes:
+ # delete the volume
+ if volume.status in ['available', 'error', 'in-use']:
+ try:
+ self.logger.info('Deleting volume {0} ({1}), currently in'
+ ' status {2} ...'.
+ format(volume.name, volume.id,
+ volume.status))
+ cinder.volumes.delete(volume)
+ except Exception as e:
+ self.logger.warning('Attempt to delete volume {0} ({1})'
+ ' yielded exception: "{2}"'.
+ format(volume.name, volume.id,
+ e))
+ unremovables[volume.id] = e
+ existing_volumes.remove(volume)
+
+ # wait for all volumes deletion until completed or timeout is reached
+ while existing_volumes and time.time() < end_time:
+ time.sleep(3)
+ for volume in existing_volumes:
+ volume_id = volume.id
+ volume_name = volume.name
+ try:
+ vol = cinder.volumes.get(volume_id)
+ if vol.status == 'deleting':
+ self.logger.debug('volume {0} ({1}) is being '
+ 'deleted...'.format(volume_name,
+ volume_id))
+ else:
+ self.logger.warning('volume {0} ({1}) is in '
+ 'unexpected status: {2}'.
+ format(volume_name, volume_id,
+ vol.status))
+ except Exception as e:
+ # the volume wasn't found, it was deleted
+ if hasattr(e, 'code') and e.code == 404:
+ self.logger.info('deleted volume {0} ({1})'.
+ format(volume_name, volume_id))
+ existing_volumes.remove(volume)
+ else:
+ self.logger.warning('failed to remove volume {0} '
+ '({1}), exception: {2}'.
+ format(volume_name,
+ volume_id, e))
+ unremovables[volume_id] = e
+ existing_volumes.remove(volume)
+
+ if existing_volumes:
+ for volume in existing_volumes:
+ # try to get the volume's status
+ try:
+ vol = cinder.volumes.get(volume.id)
+ vol_status = vol.status
+ except:
+ # failed to get volume... status is unknown
+ vol_status = 'unknown'
+
+ unremovables[volume.id] = 'timed out while removing volume '\
+ '{0} ({1}), current volume status '\
+ 'is {2}'.format(volume.name,
+ volume.id,
+ vol_status)
+
+ if unremovables:
+ self.logger.warning('failed to remove volumes: {0}'.format(
+ unremovables))
+
+ return unremovables
+
+ def _client_creds(self):
+ return {
+ 'username': self.env.keystone_username,
+ 'password': self.env.keystone_password,
+ 'auth_url': self.env.keystone_url,
+ 'project_name': self.env.keystone_tenant_name,
+ 'region_name': self.env.region
+ }
+
+ def _networks(self, neutron, prefix):
+ return [(n['id'], n['name'])
+ for n in neutron.list_networks()['networks']
+ if self._check_prefix(n['name'], prefix)]
+
+ def _subnets(self, neutron, prefix):
+ return [(n['id'], n['name'])
+ for n in neutron.list_subnets()['subnets']
+ if self._check_prefix(n['name'], prefix)]
+
+ def _routers(self, neutron, prefix):
+ return [(n['id'], n['name'])
+ for n in neutron.list_routers()['routers']
+ if self._check_prefix(n['name'], prefix)]
+
+ def _security_groups(self, neutron, prefix):
+ return [(n['id'], n['name'])
+ for n in neutron.list_security_groups()['security_groups']
+ if self._check_prefix(n['name'], prefix)]
+
+ def _servers(self, nova, prefix):
+ return [(s.id, s.human_id)
+ for s in nova.servers.list()
+ if self._check_prefix(s.human_id, prefix)]
+
+ def _key_pairs(self, nova, prefix):
+ return [(kp.id, kp.name)
+ for kp in nova.keypairs.list()
+ if self._check_prefix(kp.name, prefix)]
+
+ def _floatingips(self, neutron, prefix):
+ return [(ip['id'], ip['floating_ip_address'])
+ for ip in neutron.list_floatingips()['floatingips']]
+
+ def _ports(self, neutron, prefix):
+ return [(p['id'], p['name'])
+ for p in neutron.list_ports()['ports']
+ if self._check_prefix(p['name'], prefix)]
+
+ def _volumes(self, cinder, prefix):
+ return [(v.id, v.name) for v in cinder.volumes.list()
+ if self._check_prefix(v.name, prefix)]
+
+ def _check_prefix(self, name, prefix):
+ # some openstack resources (eg. volumes) can have no display_name,
+ # in which case it's None
+ return name is None or name.startswith(prefix)
+
+ def _remove_keys(self, dct, keys):
+ for key in keys:
+ if key in dct:
+ del dct[key]
+ return dct
+
+ @contextmanager
+ def _handled_exception(self, resource_id, failed, resource_group):
+ try:
+ yield
+ except BaseException, ex:
+ failed[resource_group][resource_id] = ex
+
+
+handler = OpenstackHandler