ARIA multivim plugin initial checkin

Change-Id: I3a24ab6fc5ba54466bfecaf596a13b8907248ae8
Issue-id: SO-77
Signed-off-by: DeWayne Filppi <dewayne@gigaspaces.com>

3 files changed, 757 insertions, 0 deletions

diff --git a/aria/multivim-plugin/system_tests/__init__.py b/aria/multivim-plugin/system_tests/__init__.py
new file mode 100644
index 0000000000..3ad9513f40
--- /dev/null
+++ b/aria/multivim-plugin/system_tests/__init__.py
@@ -0,0 +1,2 @@
+from pkgutil import extend_path
+__path__ = extend_path(__path__, __name__)
diff --git a/aria/multivim-plugin/system_tests/openstack_handler.py b/aria/multivim-plugin/system_tests/openstack_handler.py
new file mode 100644
index 0000000000..76368fa10a
--- /dev/null
+++ b/aria/multivim-plugin/system_tests/openstack_handler.py
@@ -0,0 +1,657 @@
+########
+# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    * See the License for the specific language governing permissions and
+#    * limitations under the License.
+
+import random
+import logging
+import os
+import time
+import copy
+from contextlib import contextmanager
+
+from cinderclient import client as cinderclient
+from keystoneauth1 import loading, session
+import novaclient.client as nvclient
+import neutronclient.v2_0.client as neclient
+from retrying import retry
+
+from cosmo_tester.framework.handlers import (
+    BaseHandler,
+    BaseCloudifyInputsConfigReader)
+from cosmo_tester.framework.util import get_actual_keypath
+
+logging.getLogger('neutronclient.client').setLevel(logging.INFO)
+logging.getLogger('novaclient.client').setLevel(logging.INFO)
+
+
+VOLUME_TERMINATION_TIMEOUT_SECS = 300
+
+
+class OpenstackCleanupContext(BaseHandler.CleanupContext):
+
+    def __init__(self, context_name, env):
+        super(OpenstackCleanupContext, self).__init__(context_name, env)
+        self.before_run = self.env.handler.openstack_infra_state()
+
+    def cleanup(self):
+        """
+        Cleans resources created by the test.
+        Resource that existed before the test will not be removed
+        """
+        super(OpenstackCleanupContext, self).cleanup()
+        resources_to_teardown = self.get_resources_to_teardown(
+            self.env, resources_to_keep=self.before_run)
+        if self.skip_cleanup:
+            self.logger.warn('[{0}] SKIPPING cleanup of resources: {1}'
+                             .format(self.context_name, resources_to_teardown))
+        else:
+            self._clean(self.env, resources_to_teardown)
+
+    @classmethod
+    def clean_all(cls, env):
+        """
+        Cleans *all* resources, including resources that were not
+        created by the test
+        """
+        super(OpenstackCleanupContext, cls).clean_all(env)
+        resources_to_teardown = cls.get_resources_to_teardown(env)
+        cls._clean(env, resources_to_teardown)
+
+    @classmethod
+    def _clean(cls, env, resources_to_teardown):
+        cls.logger.info('Openstack handler will try to remove these resources:'
+                        ' {0}'.format(resources_to_teardown))
+        failed_to_remove = env.handler.remove_openstack_resources(
+            resources_to_teardown)
+        if failed_to_remove:
+            trimmed_failed_to_remove = {key: value for key, value in
+                                        failed_to_remove.iteritems()
+                                        if value}
+            if len(trimmed_failed_to_remove) > 0:
+                msg = 'Openstack handler failed to remove some resources:' \
+                      ' {0}'.format(trimmed_failed_to_remove)
+                cls.logger.error(msg)
+                raise RuntimeError(msg)
+
+    @classmethod
+    def get_resources_to_teardown(cls, env, resources_to_keep=None):
+        all_existing_resources = env.handler.openstack_infra_state()
+        if resources_to_keep:
+            return env.handler.openstack_infra_state_delta(
+                before=resources_to_keep, after=all_existing_resources)
+        else:
+            return all_existing_resources
+
+    def update_server_id(self, server_name):
+
+        # retrieve the id of the new server
+        nova, _, _ = self.env.handler.openstack_clients()
+        servers = nova.servers.list(
+            search_opts={'name': server_name})
+        if len(servers) > 1:
+            raise RuntimeError(
+                'Expected 1 server with name {0}, but found {1}'
+                .format(server_name, len(servers)))
+
+        new_server_id = servers[0].id
+
+        # retrieve the id of the old server
+        old_server_id = None
+        servers = self.before_run['servers']
+        for server_id, name in servers.iteritems():
+            if server_name == name:
+                old_server_id = server_id
+                break
+        if old_server_id is None:
+            raise RuntimeError(
+                'Could not find a server with name {0} '
+                'in the internal cleanup context state'
+                .format(server_name))
+
+        # replace the id in the internal state
+        servers[new_server_id] = servers.pop(old_server_id)
+
+
+class CloudifyOpenstackInputsConfigReader(BaseCloudifyInputsConfigReader):
+
+    def __init__(self, cloudify_config, manager_blueprint_path, **kwargs):
+        super(CloudifyOpenstackInputsConfigReader, self).__init__(
+            cloudify_config, manager_blueprint_path=manager_blueprint_path,
+            **kwargs)
+
+    @property
+    def region(self):
+        return self.config['region']
+
+    @property
+    def management_server_name(self):
+        return self.config['manager_server_name']
+
+    @property
+    def agent_key_path(self):
+        return self.config['agent_private_key_path']
+
+    @property
+    def management_user_name(self):
+        return self.config['ssh_user']
+
+    @property
+    def management_key_path(self):
+        return self.config['ssh_key_filename']
+
+    @property
+    def agent_keypair_name(self):
+        return self.config['agent_public_key_name']
+
+    @property
+    def management_keypair_name(self):
+        return self.config['manager_public_key_name']
+
+    @property
+    def use_existing_agent_keypair(self):
+        return self.config['use_existing_agent_keypair']
+
+    @property
+    def use_existing_manager_keypair(self):
+        return self.config['use_existing_manager_keypair']
+
+    @property
+    def external_network_name(self):
+        return self.config['external_network_name']
+
+    @property
+    def keystone_username(self):
+        return self.config['keystone_username']
+
+    @property
+    def keystone_password(self):
+        return self.config['keystone_password']
+
+    @property
+    def keystone_tenant_name(self):
+        return self.config['keystone_tenant_name']
+
+    @property
+    def keystone_url(self):
+        return self.config['keystone_url']
+
+    @property
+    def neutron_url(self):
+        return self.config.get('neutron_url', None)
+
+    @property
+    def management_network_name(self):
+        return self.config['management_network_name']
+
+    @property
+    def management_subnet_name(self):
+        return self.config['management_subnet_name']
+
+    @property
+    def management_router_name(self):
+        return self.config['management_router']
+
+    @property
+    def agents_security_group(self):
+        return self.config['agents_security_group_name']
+
+    @property
+    def management_security_group(self):
+        return self.config['manager_security_group_name']
+
+
+class OpenstackHandler(BaseHandler):
+
+    CleanupContext = OpenstackCleanupContext
+    CloudifyConfigReader = CloudifyOpenstackInputsConfigReader
+
+    def before_bootstrap(self):
+        super(OpenstackHandler, self).before_bootstrap()
+        with self.update_cloudify_config() as patch:
+            suffix = '-%06x' % random.randrange(16 ** 6)
+            server_name_prop_path = 'manager_server_name'
+            patch.append_value(server_name_prop_path, suffix)
+
+    def after_bootstrap(self, provider_context):
+        super(OpenstackHandler, self).after_bootstrap(provider_context)
+        resources = provider_context['resources']
+        agent_keypair = resources['agents_keypair']
+        management_keypair = resources['management_keypair']
+        self.remove_agent_keypair = agent_keypair['external_resource'] is False
+        self.remove_management_keypair = \
+            management_keypair['external_resource'] is False
+
+    def after_teardown(self):
+        super(OpenstackHandler, self).after_teardown()
+        if self.remove_agent_keypair:
+            agent_key_path = get_actual_keypath(self.env,
+                                                self.env.agent_key_path,
+                                                raise_on_missing=False)
+            if agent_key_path:
+                os.remove(agent_key_path)
+        if self.remove_management_keypair:
+            management_key_path = get_actual_keypath(
+                self.env,
+                self.env.management_key_path,
+                raise_on_missing=False)
+            if management_key_path:
+                os.remove(management_key_path)
+
+    def openstack_clients(self):
+        creds = self._client_creds()
+        params = {
+            'region_name': creds.pop('region_name'),
+        }
+
+        loader = loading.get_plugin_loader("password")
+        auth = loader.load_from_options(**creds)
+        sess = session.Session(auth=auth, verify=True)
+
+        params['session'] = sess
+
+        nova = nvclient.Client('2', **params)
+        neutron = neclient.Client(**params)
+        cinder = cinderclient.Client('2', **params)
+
+        return (nova, neutron, cinder)
+
+    @retry(stop_max_attempt_number=5, wait_fixed=20000)
+    def openstack_infra_state(self):
+        """
+        @retry decorator is used because this error sometimes occur:
+        ConnectionFailed: Connection to neutron failed: Maximum
+        attempts reached
+        """
+        nova, neutron, cinder = self.openstack_clients()
+        try:
+            prefix = self.env.resources_prefix
+        except (AttributeError, KeyError):
+            prefix = ''
+        return {
+            'networks': dict(self._networks(neutron, prefix)),
+            'subnets': dict(self._subnets(neutron, prefix)),
+            'routers': dict(self._routers(neutron, prefix)),
+            'security_groups': dict(self._security_groups(neutron, prefix)),
+            'servers': dict(self._servers(nova, prefix)),
+            'key_pairs': dict(self._key_pairs(nova, prefix)),
+            'floatingips': dict(self._floatingips(neutron, prefix)),
+            'ports': dict(self._ports(neutron, prefix)),
+            'volumes': dict(self._volumes(cinder, prefix))
+        }
+
+    def openstack_infra_state_delta(self, before, after):
+        after = copy.deepcopy(after)
+        return {
+            prop: self._remove_keys(after[prop], before[prop].keys())
+            for prop in before
+        }
+
+    def _find_keypairs_to_delete(self, nodes, node_instances):
+        """Filter the nodes only returning the names of keypair nodes
+
+        Examine node_instances and nodes, return the external_name of
+        those node_instances, which correspond to a node that has a
+        type == KeyPair
+
+        To filter by deployment_id, simply make sure that the nodes and
+        node_instances this method receives, are pre-filtered
+        (ie. filter the nodes while fetching them from the manager)
+        """
+        keypairs = set()  # a set of (deployment_id, node_id) tuples
+
+        for node in nodes:
+            if node.get('type') != 'cloudify.openstack.nodes.KeyPair':
+                continue
+            # deployment_id isnt always present in local_env runs
+            key = (node.get('deployment_id'), node['id'])
+            keypairs.add(key)
+
+        for node_instance in node_instances:
+            key = (node_instance.get('deployment_id'),
+                   node_instance['node_id'])
+            if key not in keypairs:
+                continue
+
+            runtime_properties = node_instance['runtime_properties']
+            if not runtime_properties:
+                continue
+            name = runtime_properties.get('external_name')
+            if name:
+                yield name
+
+    def _delete_keypairs_by_name(self, keypair_names):
+        nova, neutron, cinder = self.openstack_clients()
+        existing_keypairs = nova.keypairs.list()
+
+        for name in keypair_names:
+            for keypair in existing_keypairs:
+                if keypair.name == name:
+                    nova.keypairs.delete(keypair)
+
+    def remove_keypairs_from_local_env(self, local_env):
+        """Query the local_env for nodes which are keypairs, remove them
+
+        Similar to querying the manager, we can look up nodes in the local_env
+        which is used for tests.
+        """
+        nodes = local_env.storage.get_nodes()
+        node_instances = local_env.storage.get_node_instances()
+        names = self._find_keypairs_to_delete(nodes, node_instances)
+        self._delete_keypairs_by_name(names)
+
+    def remove_keypairs_from_manager(self, deployment_id=None,
+                                     rest_client=None):
+        """Query the manager for nodes by deployment_id, delete keypairs
+
+        Fetch nodes and node_instances from the manager by deployment_id
+        (or all if not given), find which ones represent openstack keypairs,
+        remove them.
+        """
+        if rest_client is None:
+            rest_client = self.env.rest_client
+
+        nodes = rest_client.nodes.list(deployment_id=deployment_id)
+        node_instances = rest_client.node_instances.list(
+            deployment_id=deployment_id)
+        keypairs = self._find_keypairs_to_delete(nodes, node_instances)
+        self._delete_keypairs_by_name(keypairs)
+
+    def remove_keypair(self, name):
+        """Delete an openstack keypair by name. If it doesnt exist, do nothing.
+        """
+        self._delete_keypairs_by_name([name])
+
+    def remove_openstack_resources(self, resources_to_remove):
+        # basically sort of a workaround, but if we get the order wrong
+        # the first time, there is a chance things would better next time
+        # 3'rd time can't really hurt, can it?
+        # 3 is a charm
+        for _ in range(3):
+            resources_to_remove = self._remove_openstack_resources_impl(
+                resources_to_remove)
+            if all([len(g) == 0 for g in resources_to_remove.values()]):
+                break
+            # give openstack some time to update its data structures
+            time.sleep(3)
+        return resources_to_remove
+
+    def _remove_openstack_resources_impl(self, resources_to_remove):
+        nova, neutron, cinder = self.openstack_clients()
+
+        servers = nova.servers.list()
+        ports = neutron.list_ports()['ports']
+        routers = neutron.list_routers()['routers']
+        subnets = neutron.list_subnets()['subnets']
+        networks = neutron.list_networks()['networks']
+        # keypairs = nova.keypairs.list()
+        floatingips = neutron.list_floatingips()['floatingips']
+        security_groups = neutron.list_security_groups()['security_groups']
+        volumes = cinder.volumes.list()
+
+        failed = {
+            'servers': {},
+            'routers': {},
+            'ports': {},
+            'subnets': {},
+            'networks': {},
+            'key_pairs': {},
+            'floatingips': {},
+            'security_groups': {},
+            'volumes': {}
+        }
+
+        volumes_to_remove = []
+        for volume in volumes:
+            if volume.id in resources_to_remove['volumes']:
+                volumes_to_remove.append(volume)
+
+        left_volumes = self._delete_volumes(nova, cinder, volumes_to_remove)
+        for volume_id, ex in left_volumes.iteritems():
+            failed['volumes'][volume_id] = ex
+
+        for server in servers:
+            if server.id in resources_to_remove['servers']:
+                with self._handled_exception(server.id, failed, 'servers'):
+                    nova.servers.delete(server)
+
+        for router in routers:
+            if router['id'] in resources_to_remove['routers']:
+                with self._handled_exception(router['id'], failed, 'routers'):
+                    for p in neutron.list_ports(
+                            device_id=router['id'])['ports']:
+                        neutron.remove_interface_router(router['id'], {
+                            'port_id': p['id']
+                        })
+                    neutron.delete_router(router['id'])
+
+        for port in ports:
+            if port['id'] in resources_to_remove['ports']:
+                with self._handled_exception(port['id'], failed, 'ports'):
+                    neutron.delete_port(port['id'])
+
+        for subnet in subnets:
+            if subnet['id'] in resources_to_remove['subnets']:
+                with self._handled_exception(subnet['id'], failed, 'subnets'):
+                    neutron.delete_subnet(subnet['id'])
+
+        for network in networks:
+            if network['name'] == self.env.external_network_name:
+                continue
+            if network['id'] in resources_to_remove['networks']:
+                with self._handled_exception(network['id'], failed,
+                                             'networks'):
+                    neutron.delete_network(network['id'])
+
+        # TODO: implement key-pair creation and cleanup per tenant
+        #
+        # IMPORTANT: Do not remove key-pairs, they might be used
+        # by another tenant (of the same user)
+        #
+        # for key_pair in keypairs:
+        #     if key_pair.name == self.env.agent_keypair_name and \
+        #             self.env.use_existing_agent_keypair:
+        #             # this is a pre-existing agent key-pair, do not remove
+        #             continue
+        #     elif key_pair.name == self.env.management_keypair_name and \
+        #             self.env.use_existing_manager_keypair:
+        #             # this is a pre-existing manager key-pair, do not remove
+        #             continue
+        #     elif key_pair.id in resources_to_remove['key_pairs']:
+        #         with self._handled_exception(key_pair.id, failed,
+        #           'key_pairs'):
+        #             nova.keypairs.delete(key_pair)
+
+        for floatingip in floatingips:
+            if floatingip['id'] in resources_to_remove['floatingips']:
+                with self._handled_exception(floatingip['id'], failed,
+                                             'floatingips'):
+                    neutron.delete_floatingip(floatingip['id'])
+
+        for security_group in security_groups:
+            if security_group['name'] == 'default':
+                continue
+            if security_group['id'] in resources_to_remove['security_groups']:
+                with self._handled_exception(security_group['id'],
+                                             failed, 'security_groups'):
+                    neutron.delete_security_group(security_group['id'])
+
+        return failed
+
+    def _delete_volumes(self, nova, cinder, existing_volumes):
+        unremovables = {}
+        end_time = time.time() + VOLUME_TERMINATION_TIMEOUT_SECS
+
+        for volume in existing_volumes:
+            # detach the volume
+            if volume.status in ['available', 'error', 'in-use']:
+                try:
+                    self.logger.info('Detaching volume {0} ({1}), currently in'
+                                     ' status {2} ...'.
+                                     format(volume.name, volume.id,
+                                            volume.status))
+                    for attachment in volume.attachments:
+                        nova.volumes.delete_server_volume(
+                            server_id=attachment['server_id'],
+                            attachment_id=attachment['id'])
+                except Exception as e:
+                    self.logger.warning('Attempt to detach volume {0} ({1})'
+                                        ' yielded exception: "{2}"'.
+                                        format(volume.name, volume.id,
+                                               e))
+                    unremovables[volume.id] = e
+                    existing_volumes.remove(volume)
+
+        time.sleep(3)
+        for volume in existing_volumes:
+            # delete the volume
+            if volume.status in ['available', 'error', 'in-use']:
+                try:
+                    self.logger.info('Deleting volume {0} ({1}), currently in'
+                                     ' status {2} ...'.
+                                     format(volume.name, volume.id,
+                                            volume.status))
+                    cinder.volumes.delete(volume)
+                except Exception as e:
+                    self.logger.warning('Attempt to delete volume {0} ({1})'
+                                        ' yielded exception: "{2}"'.
+                                        format(volume.name, volume.id,
+                                               e))
+                    unremovables[volume.id] = e
+                    existing_volumes.remove(volume)
+
+        # wait for all volumes deletion until completed or timeout is reached
+        while existing_volumes and time.time() < end_time:
+            time.sleep(3)
+            for volume in existing_volumes:
+                volume_id = volume.id
+                volume_name = volume.name
+                try:
+                    vol = cinder.volumes.get(volume_id)
+                    if vol.status == 'deleting':
+                        self.logger.debug('volume {0} ({1}) is being '
+                                          'deleted...'.format(volume_name,
+                                                              volume_id))
+                    else:
+                        self.logger.warning('volume {0} ({1}) is in '
+                                            'unexpected status: {2}'.
+                                            format(volume_name, volume_id,
+                                                   vol.status))
+                except Exception as e:
+                    # the volume wasn't found, it was deleted
+                    if hasattr(e, 'code') and e.code == 404:
+                        self.logger.info('deleted volume {0} ({1})'.
+                                         format(volume_name, volume_id))
+                        existing_volumes.remove(volume)
+                    else:
+                        self.logger.warning('failed to remove volume {0} '
+                                            '({1}), exception: {2}'.
+                                            format(volume_name,
+                                                   volume_id, e))
+                        unremovables[volume_id] = e
+                        existing_volumes.remove(volume)
+
+        if existing_volumes:
+            for volume in existing_volumes:
+                # try to get the volume's status
+                try:
+                    vol = cinder.volumes.get(volume.id)
+                    vol_status = vol.status
+                except:
+                    # failed to get volume... status is unknown
+                    vol_status = 'unknown'
+
+                unremovables[volume.id] = 'timed out while removing volume '\
+                                          '{0} ({1}), current volume status '\
+                                          'is {2}'.format(volume.name,
+                                                          volume.id,
+                                                          vol_status)
+
+        if unremovables:
+            self.logger.warning('failed to remove volumes: {0}'.format(
+                unremovables))
+
+        return unremovables
+
+    def _client_creds(self):
+        return {
+            'username': self.env.keystone_username,
+            'password': self.env.keystone_password,
+            'auth_url': self.env.keystone_url,
+            'project_name': self.env.keystone_tenant_name,
+            'region_name': self.env.region
+        }
+
+    def _networks(self, neutron, prefix):
+        return [(n['id'], n['name'])
+                for n in neutron.list_networks()['networks']
+                if self._check_prefix(n['name'], prefix)]
+
+    def _subnets(self, neutron, prefix):
+        return [(n['id'], n['name'])
+                for n in neutron.list_subnets()['subnets']
+                if self._check_prefix(n['name'], prefix)]
+
+    def _routers(self, neutron, prefix):
+        return [(n['id'], n['name'])
+                for n in neutron.list_routers()['routers']
+                if self._check_prefix(n['name'], prefix)]
+
+    def _security_groups(self, neutron, prefix):
+        return [(n['id'], n['name'])
+                for n in neutron.list_security_groups()['security_groups']
+                if self._check_prefix(n['name'], prefix)]
+
+    def _servers(self, nova, prefix):
+        return [(s.id, s.human_id)
+                for s in nova.servers.list()
+                if self._check_prefix(s.human_id, prefix)]
+
+    def _key_pairs(self, nova, prefix):
+        return [(kp.id, kp.name)
+                for kp in nova.keypairs.list()
+                if self._check_prefix(kp.name, prefix)]
+
+    def _floatingips(self, neutron, prefix):
+        return [(ip['id'], ip['floating_ip_address'])
+                for ip in neutron.list_floatingips()['floatingips']]
+
+    def _ports(self, neutron, prefix):
+        return [(p['id'], p['name'])
+                for p in neutron.list_ports()['ports']
+                if self._check_prefix(p['name'], prefix)]
+
+    def _volumes(self, cinder, prefix):
+        return [(v.id, v.name) for v in cinder.volumes.list()
+                if self._check_prefix(v.name, prefix)]
+
+    def _check_prefix(self, name, prefix):
+        # some openstack resources (eg. volumes) can have no display_name,
+        # in which case it's None
+        return name is None or name.startswith(prefix)
+
+    def _remove_keys(self, dct, keys):
+        for key in keys:
+            if key in dct:
+                del dct[key]
+        return dct
+
+    @contextmanager
+    def _handled_exception(self, resource_id, failed, resource_group):
+        try:
+            yield
+        except BaseException, ex:
+            failed[resource_group][resource_id] = ex
+
+
+handler = OpenstackHandler
diff --git a/aria/multivim-plugin/system_tests/openstack_nova_net_handler.py b/aria/multivim-plugin/system_tests/openstack_nova_net_handler.py
new file mode 100644
index 0000000000..06fa0ab4d0
--- /dev/null
+++ b/aria/multivim-plugin/system_tests/openstack_nova_net_handler.py
@@ -0,0 +1,98 @@
+########
+# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    * See the License for the specific language governing permissions and
+#    * limitations under the License.
+
+import novaclient.v2.client as nvclient
+
+from system_tests.openstack_handler import OpenstackHandler
+
+
+class OpenstackNovaNetHandler(OpenstackHandler):
+
+    # using the Config Readers of the regular OpenstackHandler - attempts
+    # of reading neutron-related data may fail but shouldn't happen from
+    # nova-net tests in the first place
+    # CloudifyConfigReader = None
+
+    def openstack_clients(self):
+        creds = self._client_creds()
+        return nvclient.Client(**creds)
+
+    def openstack_infra_state(self):
+        nova = self.openstack_clients()
+        prefix = self.env.resources_prefix
+        return {
+            'security_groups': dict(self._security_groups(nova, prefix)),
+            'servers': dict(self._servers(nova, prefix)),
+            'key_pairs': dict(self._key_pairs(nova, prefix)),
+            'floatingips': dict(self._floatingips(nova, prefix)),
+        }
+
+    def _floatingips(self, nova, prefix):
+        return [(ip.id, ip.ip)
+                for ip in nova.floating_ips.list()]
+
+    def _security_groups(self, nova, prefix):
+        return [(n.id, n.name)
+                for n in nova.security_groups.list()
+                if self._check_prefix(n.name, prefix)]
+
+    def _remove_openstack_resources_impl(self, resources_to_remove):
+        nova = self.openstack_clients()
+
+        servers = nova.servers.list()
+        keypairs = nova.keypairs.list()
+        floatingips = nova.floating_ips.list()
+        security_groups = nova.security_groups.list()
+
+        failed = {
+            'servers': {},
+            'key_pairs': {},
+            'floatingips': {},
+            'security_groups': {}
+        }
+
+        for server in servers:
+            if server.id in resources_to_remove['servers']:
+                with self._handled_exception(server.id, failed, 'servers'):
+                    nova.servers.delete(server)
+        for key_pair in keypairs:
+            if key_pair.name == self.env.agent_keypair_name and \
+                    self.env.use_existing_agent_keypair:
+                # this is a pre-existing agent key-pair, do not remove
+                continue
+            elif key_pair.name == self.env.management_keypair_name and \
+                    self.env.use_existing_manager_keypair:
+                # this is a pre-existing manager key-pair, do not remove
+                continue
+            elif key_pair.id in resources_to_remove['key_pairs']:
+                with self._handled_exception(key_pair.id, failed, 'key_pairs'):
+                    nova.keypairs.delete(key_pair)
+        for floatingip in floatingips:
+            if floatingip.id in resources_to_remove['floatingips']:
+                with self._handled_exception(floatingip.id, failed,
+                                             'floatingips'):
+                    nova.floating_ips.delete(floatingip)
+        for security_group in security_groups:
+            if security_group.name == 'default':
+                continue
+            if security_group.id in resources_to_remove['security_groups']:
+                with self._handled_exception(security_group.id, failed,
+                                             'security_groups'):
+                    nova.security_groups.delete(security_group)
+
+        return failed
+
+
+handler = OpenstackNovaNetHandler