reorg and add pom build for wheel

Change-Id: Iab761e263f1e2380471dd38c2b7ce9b77f0aab0e
Signed-off-by: DeWayne Filppi <dewayne@cloudify.co>
Issue-id: SO-106

1 files changed, 0 insertions, 657 deletions

diff --git a/aria/multivim-plugin/system_tests/openstack_handler.py b/aria/multivim-plugin/system_tests/openstack_handler.py
deleted file mode 100644
index 76368fa10a..0000000000
--- a/aria/multivim-plugin/system_tests/openstack_handler.py
+++ /dev/null
@@ -1,657 +0,0 @@
-########
-# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    * See the License for the specific language governing permissions and
-#    * limitations under the License.
-
-import random
-import logging
-import os
-import time
-import copy
-from contextlib import contextmanager
-
-from cinderclient import client as cinderclient
-from keystoneauth1 import loading, session
-import novaclient.client as nvclient
-import neutronclient.v2_0.client as neclient
-from retrying import retry
-
-from cosmo_tester.framework.handlers import (
-    BaseHandler,
-    BaseCloudifyInputsConfigReader)
-from cosmo_tester.framework.util import get_actual_keypath
-
-logging.getLogger('neutronclient.client').setLevel(logging.INFO)
-logging.getLogger('novaclient.client').setLevel(logging.INFO)
-
-
-VOLUME_TERMINATION_TIMEOUT_SECS = 300
-
-
-class OpenstackCleanupContext(BaseHandler.CleanupContext):
-
-    def __init__(self, context_name, env):
-        super(OpenstackCleanupContext, self).__init__(context_name, env)
-        self.before_run = self.env.handler.openstack_infra_state()
-
-    def cleanup(self):
-        """
-        Cleans resources created by the test.
-        Resource that existed before the test will not be removed
-        """
-        super(OpenstackCleanupContext, self).cleanup()
-        resources_to_teardown = self.get_resources_to_teardown(
-            self.env, resources_to_keep=self.before_run)
-        if self.skip_cleanup:
-            self.logger.warn('[{0}] SKIPPING cleanup of resources: {1}'
-                             .format(self.context_name, resources_to_teardown))
-        else:
-            self._clean(self.env, resources_to_teardown)
-
-    @classmethod
-    def clean_all(cls, env):
-        """
-        Cleans *all* resources, including resources that were not
-        created by the test
-        """
-        super(OpenstackCleanupContext, cls).clean_all(env)
-        resources_to_teardown = cls.get_resources_to_teardown(env)
-        cls._clean(env, resources_to_teardown)
-
-    @classmethod
-    def _clean(cls, env, resources_to_teardown):
-        cls.logger.info('Openstack handler will try to remove these resources:'
-                        ' {0}'.format(resources_to_teardown))
-        failed_to_remove = env.handler.remove_openstack_resources(
-            resources_to_teardown)
-        if failed_to_remove:
-            trimmed_failed_to_remove = {key: value for key, value in
-                                        failed_to_remove.iteritems()
-                                        if value}
-            if len(trimmed_failed_to_remove) > 0:
-                msg = 'Openstack handler failed to remove some resources:' \
-                      ' {0}'.format(trimmed_failed_to_remove)
-                cls.logger.error(msg)
-                raise RuntimeError(msg)
-
-    @classmethod
-    def get_resources_to_teardown(cls, env, resources_to_keep=None):
-        all_existing_resources = env.handler.openstack_infra_state()
-        if resources_to_keep:
-            return env.handler.openstack_infra_state_delta(
-                before=resources_to_keep, after=all_existing_resources)
-        else:
-            return all_existing_resources
-
-    def update_server_id(self, server_name):
-
-        # retrieve the id of the new server
-        nova, _, _ = self.env.handler.openstack_clients()
-        servers = nova.servers.list(
-            search_opts={'name': server_name})
-        if len(servers) > 1:
-            raise RuntimeError(
-                'Expected 1 server with name {0}, but found {1}'
-                .format(server_name, len(servers)))
-
-        new_server_id = servers[0].id
-
-        # retrieve the id of the old server
-        old_server_id = None
-        servers = self.before_run['servers']
-        for server_id, name in servers.iteritems():
-            if server_name == name:
-                old_server_id = server_id
-                break
-        if old_server_id is None:
-            raise RuntimeError(
-                'Could not find a server with name {0} '
-                'in the internal cleanup context state'
-                .format(server_name))
-
-        # replace the id in the internal state
-        servers[new_server_id] = servers.pop(old_server_id)
-
-
-class CloudifyOpenstackInputsConfigReader(BaseCloudifyInputsConfigReader):
-
-    def __init__(self, cloudify_config, manager_blueprint_path, **kwargs):
-        super(CloudifyOpenstackInputsConfigReader, self).__init__(
-            cloudify_config, manager_blueprint_path=manager_blueprint_path,
-            **kwargs)
-
-    @property
-    def region(self):
-        return self.config['region']
-
-    @property
-    def management_server_name(self):
-        return self.config['manager_server_name']
-
-    @property
-    def agent_key_path(self):
-        return self.config['agent_private_key_path']
-
-    @property
-    def management_user_name(self):
-        return self.config['ssh_user']
-
-    @property
-    def management_key_path(self):
-        return self.config['ssh_key_filename']
-
-    @property
-    def agent_keypair_name(self):
-        return self.config['agent_public_key_name']
-
-    @property
-    def management_keypair_name(self):
-        return self.config['manager_public_key_name']
-
-    @property
-    def use_existing_agent_keypair(self):
-        return self.config['use_existing_agent_keypair']
-
-    @property
-    def use_existing_manager_keypair(self):
-        return self.config['use_existing_manager_keypair']
-
-    @property
-    def external_network_name(self):
-        return self.config['external_network_name']
-
-    @property
-    def keystone_username(self):
-        return self.config['keystone_username']
-
-    @property
-    def keystone_password(self):
-        return self.config['keystone_password']
-
-    @property
-    def keystone_tenant_name(self):
-        return self.config['keystone_tenant_name']
-
-    @property
-    def keystone_url(self):
-        return self.config['keystone_url']
-
-    @property
-    def neutron_url(self):
-        return self.config.get('neutron_url', None)
-
-    @property
-    def management_network_name(self):
-        return self.config['management_network_name']
-
-    @property
-    def management_subnet_name(self):
-        return self.config['management_subnet_name']
-
-    @property
-    def management_router_name(self):
-        return self.config['management_router']
-
-    @property
-    def agents_security_group(self):
-        return self.config['agents_security_group_name']
-
-    @property
-    def management_security_group(self):
-        return self.config['manager_security_group_name']
-
-
-class OpenstackHandler(BaseHandler):
-
-    CleanupContext = OpenstackCleanupContext
-    CloudifyConfigReader = CloudifyOpenstackInputsConfigReader
-
-    def before_bootstrap(self):
-        super(OpenstackHandler, self).before_bootstrap()
-        with self.update_cloudify_config() as patch:
-            suffix = '-%06x' % random.randrange(16 ** 6)
-            server_name_prop_path = 'manager_server_name'
-            patch.append_value(server_name_prop_path, suffix)
-
-    def after_bootstrap(self, provider_context):
-        super(OpenstackHandler, self).after_bootstrap(provider_context)
-        resources = provider_context['resources']
-        agent_keypair = resources['agents_keypair']
-        management_keypair = resources['management_keypair']
-        self.remove_agent_keypair = agent_keypair['external_resource'] is False
-        self.remove_management_keypair = \
-            management_keypair['external_resource'] is False
-
-    def after_teardown(self):
-        super(OpenstackHandler, self).after_teardown()
-        if self.remove_agent_keypair:
-            agent_key_path = get_actual_keypath(self.env,
-                                                self.env.agent_key_path,
-                                                raise_on_missing=False)
-            if agent_key_path:
-                os.remove(agent_key_path)
-        if self.remove_management_keypair:
-            management_key_path = get_actual_keypath(
-                self.env,
-                self.env.management_key_path,
-                raise_on_missing=False)
-            if management_key_path:
-                os.remove(management_key_path)
-
-    def openstack_clients(self):
-        creds = self._client_creds()
-        params = {
-            'region_name': creds.pop('region_name'),
-        }
-
-        loader = loading.get_plugin_loader("password")
-        auth = loader.load_from_options(**creds)
-        sess = session.Session(auth=auth, verify=True)
-
-        params['session'] = sess
-
-        nova = nvclient.Client('2', **params)
-        neutron = neclient.Client(**params)
-        cinder = cinderclient.Client('2', **params)
-
-        return (nova, neutron, cinder)
-
-    @retry(stop_max_attempt_number=5, wait_fixed=20000)
-    def openstack_infra_state(self):
-        """
-        @retry decorator is used because this error sometimes occur:
-        ConnectionFailed: Connection to neutron failed: Maximum
-        attempts reached
-        """
-        nova, neutron, cinder = self.openstack_clients()
-        try:
-            prefix = self.env.resources_prefix
-        except (AttributeError, KeyError):
-            prefix = ''
-        return {
-            'networks': dict(self._networks(neutron, prefix)),
-            'subnets': dict(self._subnets(neutron, prefix)),
-            'routers': dict(self._routers(neutron, prefix)),
-            'security_groups': dict(self._security_groups(neutron, prefix)),
-            'servers': dict(self._servers(nova, prefix)),
-            'key_pairs': dict(self._key_pairs(nova, prefix)),
-            'floatingips': dict(self._floatingips(neutron, prefix)),
-            'ports': dict(self._ports(neutron, prefix)),
-            'volumes': dict(self._volumes(cinder, prefix))
-        }
-
-    def openstack_infra_state_delta(self, before, after):
-        after = copy.deepcopy(after)
-        return {
-            prop: self._remove_keys(after[prop], before[prop].keys())
-            for prop in before
-        }
-
-    def _find_keypairs_to_delete(self, nodes, node_instances):
-        """Filter the nodes only returning the names of keypair nodes
-
-        Examine node_instances and nodes, return the external_name of
-        those node_instances, which correspond to a node that has a
-        type == KeyPair
-
-        To filter by deployment_id, simply make sure that the nodes and
-        node_instances this method receives, are pre-filtered
-        (ie. filter the nodes while fetching them from the manager)
-        """
-        keypairs = set()  # a set of (deployment_id, node_id) tuples
-
-        for node in nodes:
-            if node.get('type') != 'cloudify.openstack.nodes.KeyPair':
-                continue
-            # deployment_id isnt always present in local_env runs
-            key = (node.get('deployment_id'), node['id'])
-            keypairs.add(key)
-
-        for node_instance in node_instances:
-            key = (node_instance.get('deployment_id'),
-                   node_instance['node_id'])
-            if key not in keypairs:
-                continue
-
-            runtime_properties = node_instance['runtime_properties']
-            if not runtime_properties:
-                continue
-            name = runtime_properties.get('external_name')
-            if name:
-                yield name
-
-    def _delete_keypairs_by_name(self, keypair_names):
-        nova, neutron, cinder = self.openstack_clients()
-        existing_keypairs = nova.keypairs.list()
-
-        for name in keypair_names:
-            for keypair in existing_keypairs:
-                if keypair.name == name:
-                    nova.keypairs.delete(keypair)
-
-    def remove_keypairs_from_local_env(self, local_env):
-        """Query the local_env for nodes which are keypairs, remove them
-
-        Similar to querying the manager, we can look up nodes in the local_env
-        which is used for tests.
-        """
-        nodes = local_env.storage.get_nodes()
-        node_instances = local_env.storage.get_node_instances()
-        names = self._find_keypairs_to_delete(nodes, node_instances)
-        self._delete_keypairs_by_name(names)
-
-    def remove_keypairs_from_manager(self, deployment_id=None,
-                                     rest_client=None):
-        """Query the manager for nodes by deployment_id, delete keypairs
-
-        Fetch nodes and node_instances from the manager by deployment_id
-        (or all if not given), find which ones represent openstack keypairs,
-        remove them.
-        """
-        if rest_client is None:
-            rest_client = self.env.rest_client
-
-        nodes = rest_client.nodes.list(deployment_id=deployment_id)
-        node_instances = rest_client.node_instances.list(
-            deployment_id=deployment_id)
-        keypairs = self._find_keypairs_to_delete(nodes, node_instances)
-        self._delete_keypairs_by_name(keypairs)
-
-    def remove_keypair(self, name):
-        """Delete an openstack keypair by name. If it doesnt exist, do nothing.
-        """
-        self._delete_keypairs_by_name([name])
-
-    def remove_openstack_resources(self, resources_to_remove):
-        # basically sort of a workaround, but if we get the order wrong
-        # the first time, there is a chance things would better next time
-        # 3'rd time can't really hurt, can it?
-        # 3 is a charm
-        for _ in range(3):
-            resources_to_remove = self._remove_openstack_resources_impl(
-                resources_to_remove)
-            if all([len(g) == 0 for g in resources_to_remove.values()]):
-                break
-            # give openstack some time to update its data structures
-            time.sleep(3)
-        return resources_to_remove
-
-    def _remove_openstack_resources_impl(self, resources_to_remove):
-        nova, neutron, cinder = self.openstack_clients()
-
-        servers = nova.servers.list()
-        ports = neutron.list_ports()['ports']
-        routers = neutron.list_routers()['routers']
-        subnets = neutron.list_subnets()['subnets']
-        networks = neutron.list_networks()['networks']
-        # keypairs = nova.keypairs.list()
-        floatingips = neutron.list_floatingips()['floatingips']
-        security_groups = neutron.list_security_groups()['security_groups']
-        volumes = cinder.volumes.list()
-
-        failed = {
-            'servers': {},
-            'routers': {},
-            'ports': {},
-            'subnets': {},
-            'networks': {},
-            'key_pairs': {},
-            'floatingips': {},
-            'security_groups': {},
-            'volumes': {}
-        }
-
-        volumes_to_remove = []
-        for volume in volumes:
-            if volume.id in resources_to_remove['volumes']:
-                volumes_to_remove.append(volume)
-
-        left_volumes = self._delete_volumes(nova, cinder, volumes_to_remove)
-        for volume_id, ex in left_volumes.iteritems():
-            failed['volumes'][volume_id] = ex
-
-        for server in servers:
-            if server.id in resources_to_remove['servers']:
-                with self._handled_exception(server.id, failed, 'servers'):
-                    nova.servers.delete(server)
-
-        for router in routers:
-            if router['id'] in resources_to_remove['routers']:
-                with self._handled_exception(router['id'], failed, 'routers'):
-                    for p in neutron.list_ports(
-                            device_id=router['id'])['ports']:
-                        neutron.remove_interface_router(router['id'], {
-                            'port_id': p['id']
-                        })
-                    neutron.delete_router(router['id'])
-
-        for port in ports:
-            if port['id'] in resources_to_remove['ports']:
-                with self._handled_exception(port['id'], failed, 'ports'):
-                    neutron.delete_port(port['id'])
-
-        for subnet in subnets:
-            if subnet['id'] in resources_to_remove['subnets']:
-                with self._handled_exception(subnet['id'], failed, 'subnets'):
-                    neutron.delete_subnet(subnet['id'])
-
-        for network in networks:
-            if network['name'] == self.env.external_network_name:
-                continue
-            if network['id'] in resources_to_remove['networks']:
-                with self._handled_exception(network['id'], failed,
-                                             'networks'):
-                    neutron.delete_network(network['id'])
-
-        # TODO: implement key-pair creation and cleanup per tenant
-        #
-        # IMPORTANT: Do not remove key-pairs, they might be used
-        # by another tenant (of the same user)
-        #
-        # for key_pair in keypairs:
-        #     if key_pair.name == self.env.agent_keypair_name and \
-        #             self.env.use_existing_agent_keypair:
-        #             # this is a pre-existing agent key-pair, do not remove
-        #             continue
-        #     elif key_pair.name == self.env.management_keypair_name and \
-        #             self.env.use_existing_manager_keypair:
-        #             # this is a pre-existing manager key-pair, do not remove
-        #             continue
-        #     elif key_pair.id in resources_to_remove['key_pairs']:
-        #         with self._handled_exception(key_pair.id, failed,
-        #           'key_pairs'):
-        #             nova.keypairs.delete(key_pair)
-
-        for floatingip in floatingips:
-            if floatingip['id'] in resources_to_remove['floatingips']:
-                with self._handled_exception(floatingip['id'], failed,
-                                             'floatingips'):
-                    neutron.delete_floatingip(floatingip['id'])
-
-        for security_group in security_groups:
-            if security_group['name'] == 'default':
-                continue
-            if security_group['id'] in resources_to_remove['security_groups']:
-                with self._handled_exception(security_group['id'],
-                                             failed, 'security_groups'):
-                    neutron.delete_security_group(security_group['id'])
-
-        return failed
-
-    def _delete_volumes(self, nova, cinder, existing_volumes):
-        unremovables = {}
-        end_time = time.time() + VOLUME_TERMINATION_TIMEOUT_SECS
-
-        for volume in existing_volumes:
-            # detach the volume
-            if volume.status in ['available', 'error', 'in-use']:
-                try:
-                    self.logger.info('Detaching volume {0} ({1}), currently in'
-                                     ' status {2} ...'.
-                                     format(volume.name, volume.id,
-                                            volume.status))
-                    for attachment in volume.attachments:
-                        nova.volumes.delete_server_volume(
-                            server_id=attachment['server_id'],
-                            attachment_id=attachment['id'])
-                except Exception as e:
-                    self.logger.warning('Attempt to detach volume {0} ({1})'
-                                        ' yielded exception: "{2}"'.
-                                        format(volume.name, volume.id,
-                                               e))
-                    unremovables[volume.id] = e
-                    existing_volumes.remove(volume)
-
-        time.sleep(3)
-        for volume in existing_volumes:
-            # delete the volume
-            if volume.status in ['available', 'error', 'in-use']:
-                try:
-                    self.logger.info('Deleting volume {0} ({1}), currently in'
-                                     ' status {2} ...'.
-                                     format(volume.name, volume.id,
-                                            volume.status))
-                    cinder.volumes.delete(volume)
-                except Exception as e:
-                    self.logger.warning('Attempt to delete volume {0} ({1})'
-                                        ' yielded exception: "{2}"'.
-                                        format(volume.name, volume.id,
-                                               e))
-                    unremovables[volume.id] = e
-                    existing_volumes.remove(volume)
-
-        # wait for all volumes deletion until completed or timeout is reached
-        while existing_volumes and time.time() < end_time:
-            time.sleep(3)
-            for volume in existing_volumes:
-                volume_id = volume.id
-                volume_name = volume.name
-                try:
-                    vol = cinder.volumes.get(volume_id)
-                    if vol.status == 'deleting':
-                        self.logger.debug('volume {0} ({1}) is being '
-                                          'deleted...'.format(volume_name,
-                                                              volume_id))
-                    else:
-                        self.logger.warning('volume {0} ({1}) is in '
-                                            'unexpected status: {2}'.
-                                            format(volume_name, volume_id,
-                                                   vol.status))
-                except Exception as e:
-                    # the volume wasn't found, it was deleted
-                    if hasattr(e, 'code') and e.code == 404:
-                        self.logger.info('deleted volume {0} ({1})'.
-                                         format(volume_name, volume_id))
-                        existing_volumes.remove(volume)
-                    else:
-                        self.logger.warning('failed to remove volume {0} '
-                                            '({1}), exception: {2}'.
-                                            format(volume_name,
-                                                   volume_id, e))
-                        unremovables[volume_id] = e
-                        existing_volumes.remove(volume)
-
-        if existing_volumes:
-            for volume in existing_volumes:
-                # try to get the volume's status
-                try:
-                    vol = cinder.volumes.get(volume.id)
-                    vol_status = vol.status
-                except:
-                    # failed to get volume... status is unknown
-                    vol_status = 'unknown'
-
-                unremovables[volume.id] = 'timed out while removing volume '\
-                                          '{0} ({1}), current volume status '\
-                                          'is {2}'.format(volume.name,
-                                                          volume.id,
-                                                          vol_status)
-
-        if unremovables:
-            self.logger.warning('failed to remove volumes: {0}'.format(
-                unremovables))
-
-        return unremovables
-
-    def _client_creds(self):
-        return {
-            'username': self.env.keystone_username,
-            'password': self.env.keystone_password,
-            'auth_url': self.env.keystone_url,
-            'project_name': self.env.keystone_tenant_name,
-            'region_name': self.env.region
-        }
-
-    def _networks(self, neutron, prefix):
-        return [(n['id'], n['name'])
-                for n in neutron.list_networks()['networks']
-                if self._check_prefix(n['name'], prefix)]
-
-    def _subnets(self, neutron, prefix):
-        return [(n['id'], n['name'])
-                for n in neutron.list_subnets()['subnets']
-                if self._check_prefix(n['name'], prefix)]
-
-    def _routers(self, neutron, prefix):
-        return [(n['id'], n['name'])
-                for n in neutron.list_routers()['routers']
-                if self._check_prefix(n['name'], prefix)]
-
-    def _security_groups(self, neutron, prefix):
-        return [(n['id'], n['name'])
-                for n in neutron.list_security_groups()['security_groups']
-                if self._check_prefix(n['name'], prefix)]
-
-    def _servers(self, nova, prefix):
-        return [(s.id, s.human_id)
-                for s in nova.servers.list()
-                if self._check_prefix(s.human_id, prefix)]
-
-    def _key_pairs(self, nova, prefix):
-        return [(kp.id, kp.name)
-                for kp in nova.keypairs.list()
-                if self._check_prefix(kp.name, prefix)]
-
-    def _floatingips(self, neutron, prefix):
-        return [(ip['id'], ip['floating_ip_address'])
-                for ip in neutron.list_floatingips()['floatingips']]
-
-    def _ports(self, neutron, prefix):
-        return [(p['id'], p['name'])
-                for p in neutron.list_ports()['ports']
-                if self._check_prefix(p['name'], prefix)]
-
-    def _volumes(self, cinder, prefix):
-        return [(v.id, v.name) for v in cinder.volumes.list()
-                if self._check_prefix(v.name, prefix)]
-
-    def _check_prefix(self, name, prefix):
-        # some openstack resources (eg. volumes) can have no display_name,
-        # in which case it's None
-        return name is None or name.startswith(prefix)
-
-    def _remove_keys(self, dct, keys):
-        for key in keys:
-            if key in dct:
-                del dct[key]
-        return dct
-
-    @contextmanager
-    def _handled_exception(self, resource_id, failed, resource_group):
-        try:
-            yield
-        except BaseException, ex:
-            failed[resource_group][resource_id] = ex
-
-
-handler = OpenstackHandler