diff options
Diffstat (limited to 'vnftest')
| -rw-r--r-- | vnftest/common/constants.py | 28 | ||||
| -rw-r--r-- | vnftest/common/openstack_utils.py | 1196 | ||||
| -rw-r--r-- | vnftest/common/utils.py | 46 | ||||
| -rw-r--r-- | vnftest/contexts/base.py | 52 | ||||
| -rw-r--r-- | vnftest/contexts/heat.py | 591 | ||||
| -rw-r--r-- | vnftest/contexts/model.py | 433 | ||||
| -rw-r--r-- | vnftest/core/task.py | 26 | ||||
| -rw-r--r-- | vnftest/orchestrator/__init__.py | 0 | ||||
| -rw-r--r-- | vnftest/orchestrator/heat.py | 661 | ||||
| -rw-r--r-- | vnftest/tests/unit/common/test_utils.py | 13 |
10 files changed, 2433 insertions, 613 deletions
diff --git a/vnftest/common/constants.py b/vnftest/common/constants.py index 9634708..a613ff0 100644 --- a/vnftest/common/constants.py +++ b/vnftest/common/constants.py @@ -61,26 +61,6 @@ def get_param(key, default=''): raise return default - -try: - SERVER_IP = get_param('api.server_ip') -except KeyError: - try: - from pyroute2 import IPDB - except ImportError: - SERVER_IP = '172.17.0.1' - else: - with IPDB() as ip: - try: - SERVER_IP = ip.routes['default'].gateway - except KeyError: - # during unittests ip.routes['default'] can be invalid - SERVER_IP = '127.0.0.1' - -if not SERVER_IP: - SERVER_IP = '127.0.0.1' - - # dir CONF_DIR = get_param('dir.conf', '/etc/vnftest') IMAGE_DIR = get_param('dir.images', join(VNFTEST_ROOT_PATH, '../../images/')) @@ -111,6 +91,14 @@ BASE_URL = 'http://localhost:5000' ENV_ACTION_API = BASE_URL + '/vnftest/env/action' ASYNC_TASK_API = BASE_URL + '/vnftest/asynctask' +# flags +IS_EXISTING = 'is_existing' +IS_PUBLIC = 'is_public' + # general TESTCASE_PRE = 'onap_vnftest_' TESTSUITE_PRE = 'onap_' + + +# OpenStack cloud default config parameters +OS_CLOUD_DEFAULT_CONFIG = {'verify': False} diff --git a/vnftest/common/openstack_utils.py b/vnftest/common/openstack_utils.py index 1bbdc43..048a9d9 100644 --- a/vnftest/common/openstack_utils.py +++ b/vnftest/common/openstack_utils.py @@ -14,60 +14,49 @@ # vnftest comment: this is a modified copy of # yardstick/common/openstack_utils.py -from __future__ import absolute_import - -import os -import time -import sys +import copy import logging +import os -from keystoneauth1 import loading -from keystoneauth1 import session from cinderclient import client as cinderclient from novaclient import client as novaclient from glanceclient import client as glanceclient +from keystoneauth1 import loading +from keystoneauth1 import session from neutronclient.neutron import client as neutronclient -from heatclient.client import Client as heatclient +import shade +from shade import exc + +from vnftest.common import constants + log = logging.getLogger(__name__) DEFAULT_HEAT_API_VERSION = '1' DEFAULT_API_VERSION = '2' -creds = {} - # ********************************************* # CREDENTIALS # ********************************************* -def initialize(openstack_env_config): - keystone_api_version = openstack_env_config.get('OS_IDENTITY_API_VERSION', None) - - if keystone_api_version is None or keystone_api_version == '2': - keystone_v3 = False - creds['tenant_name'] = openstack_env_config['OS_TENANT_NAME'] - else: - keystone_v3 = True - creds['tenant_name'] = openstack_env_config['OS_PROJECT_NAME'] - creds['project_name'] = openstack_env_config['OS_PROJECT_NAME'] - - creds["username"] = openstack_env_config["OS_USERNAME"] - creds["password"] = openstack_env_config["OS_PASSWORD"] - creds["auth_url"] = openstack_env_config["OS_AUTH_URL"] - creds["tenant_id"] = openstack_env_config["OS_TENANT_ID"] - - if keystone_v3: - if 'OS_USER_DOMAIN_NAME' in openstack_env_config: - creds.update({ - "user_domain_name": openstack_env_config['OS_USER_DOMAIN_NAME'] - }) - if 'OS_PROJECT_DOMAIN_NAME' in openstack_env_config: - creds.update({ - "project_domain_name": openstack_env_config['OS_PROJECT_DOMAIN_NAME'] - }) - - def get_credentials(): + """Returns a creds dictionary filled with parsed from env + + Keystone API version used is 3; v2 was deprecated in 2014 (Icehouse). Along + with this deprecation, environment variable 'OS_TENANT_NAME' is replaced by + 'OS_PROJECT_NAME'. + """ + creds = {'username': os.environ.get('OS_USERNAME'), + 'password': os.environ.get('OS_PASSWORD'), + 'auth_url': os.environ.get('OS_AUTH_URL'), + 'project_name': os.environ.get('OS_PROJECT_NAME') + } + + if os.getenv('OS_USER_DOMAIN_NAME'): + creds['user_domain_name'] = os.getenv('OS_USER_DOMAIN_NAME') + if os.getenv('OS_PROJECT_DOMAIN_NAME'): + creds['project_domain_name'] = os.getenv('OS_PROJECT_DOMAIN_NAME') + return creds @@ -159,11 +148,6 @@ def get_neutron_client(): # pragma: no cover return neutronclient.Client(get_neutron_client_version(), session=sess) -def get_heat_client(): # pragma: no cover - sess = get_session() - return heatclient(get_heat_api_version(), session=sess) - - def get_glance_client_version(): # pragma: no cover try: api_version = os.environ['OS_IMAGE_API_VERSION'] @@ -179,225 +163,226 @@ def get_glance_client(): # pragma: no cover return glanceclient.Client(get_glance_client_version(), session=sess) -# ********************************************* -# NOVA -# ********************************************* -def get_instances(nova_client): # pragma: no cover - try: - return nova_client.servers.list(search_opts={'all_tenants': 1}) - except Exception: - log.exception("Error [get_instances(nova_client)]") - - -def get_instance_status(nova_client, instance): # pragma: no cover - try: - return nova_client.servers.get(instance.id).status - except Exception: - log.exception("Error [get_instance_status(nova_client)]") - - -def get_instance_by_name(nova_client, instance_name): # pragma: no cover - try: - return nova_client.servers.find(name=instance_name) - except Exception: - log.exception("Error [get_instance_by_name(nova_client, '%s')]", - instance_name) - - -def get_instance_by_id(instance_id): # pragma: no cover - try: - return get_nova_client().servers.find(id=instance_id) - except Exception: - log.exception("Error [get_instance_by_id(nova_client, '%s')]", - instance_id) - +def get_shade_client(**os_cloud_config): + """Get Shade OpenStack cloud client -def get_aggregates(nova_client): # pragma: no cover - try: - return nova_client.aggregates.list() - except Exception: - log.exception("Error [get_aggregates(nova_client)]") + By default, the input parameters given to "shade.openstack_cloud" method + are stored in "constants.OS_CLOUD_DEFAULT_CONFIG". The input parameters + passed in this function, "os_cloud_config", will overwrite the default + ones. + :param os_cloud_config: (kwargs) input arguments for + "shade.openstack_cloud" method. + :return: ``shade.OpenStackCloud`` object. + """ + params = copy.deepcopy(constants.OS_CLOUD_DEFAULT_CONFIG) + params.update(os_cloud_config) + return shade.openstack_cloud(**params) -def get_availability_zones(nova_client): # pragma: no cover - try: - return nova_client.availability_zones.list() - except Exception: - log.exception("Error [get_availability_zones(nova_client)]") - +def get_shade_operator_client(**os_cloud_config): + """Get Shade Operator cloud client -def get_availability_zone_names(nova_client): # pragma: no cover - try: - return [az.zoneName for az in get_availability_zones(nova_client)] - except Exception: - log.exception("Error [get_availability_zone_names(nova_client)]") + :return: ``shade.OperatorCloud`` object. + """ + params = copy.deepcopy(constants.OS_CLOUD_DEFAULT_CONFIG) + params.update(os_cloud_config) + return shade.operator_cloud(**params) -def create_aggregate(nova_client, aggregate_name, av_zone): # pragma: no cover - try: - nova_client.aggregates.create(aggregate_name, av_zone) - except Exception: - log.exception("Error [create_aggregate(nova_client, %s, %s)]", - aggregate_name, av_zone) - return False - else: - return True - - -def get_aggregate_id(nova_client, aggregate_name): # pragma: no cover - try: - aggregates = get_aggregates(nova_client) - _id = next((ag.id for ag in aggregates if ag.name == aggregate_name)) - except Exception: - log.exception("Error [get_aggregate_id(nova_client, %s)]", - aggregate_name) - else: - return _id - - -def add_host_to_aggregate(nova_client, aggregate_name, - compute_host): # pragma: no cover - try: - aggregate_id = get_aggregate_id(nova_client, aggregate_name) - nova_client.aggregates.add_host(aggregate_id, compute_host) - except Exception: - log.exception("Error [add_host_to_aggregate(nova_client, %s, %s)]", - aggregate_name, compute_host) - return False - else: - return True - - -def create_aggregate_with_host(nova_client, aggregate_name, av_zone, - compute_host): # pragma: no cover - try: - create_aggregate(nova_client, aggregate_name, av_zone) - add_host_to_aggregate(nova_client, aggregate_name, compute_host) - except Exception: - log.exception("Error [create_aggregate_with_host(" - "nova_client, %s, %s, %s)]", - aggregate_name, av_zone, compute_host) - return False - else: - return True - - -def create_keypair(nova_client, name, key_path=None): # pragma: no cover - try: - with open(key_path) as fpubkey: - keypair = get_nova_client().keypairs.create(name=name, public_key=fpubkey.read()) - return keypair - except Exception: - log.exception("Error [create_keypair(nova_client)]") - - -def create_instance(json_body): # pragma: no cover - try: - return get_nova_client().servers.create(**json_body) - except Exception: - log.exception("Error create instance failed") - return None - - -def create_instance_and_wait_for_active(json_body): # pragma: no cover - SLEEP = 3 - VM_BOOT_TIMEOUT = 180 - nova_client = get_nova_client() - instance = create_instance(json_body) - count = VM_BOOT_TIMEOUT / SLEEP - for n in range(count, -1, -1): - status = get_instance_status(nova_client, instance) - if status.lower() == "active": - return instance - elif status.lower() == "error": - log.error("The instance went to ERROR status.") - return None - time.sleep(SLEEP) - log.error("Timeout booting the instance.") - return None - - -def attach_server_volume(server_id, volume_id, device=None): # pragma: no cover - try: - get_nova_client().volumes.create_server_volume(server_id, volume_id, device) - except Exception: - log.exception("Error [attach_server_volume(nova_client, '%s', '%s')]", - server_id, volume_id) - return False - else: +# ********************************************* +# NOVA +# ********************************************* +def create_keypair(shade_client, name, public_key=None): + """Create a new keypair. + + :param name: Name of the keypair being created. + :param public_key: Public key for the new keypair. + + :return: Created keypair. + """ + try: + return shade_client.create_keypair(name, public_key=public_key) + except exc.OpenStackCloudException as o_exc: + log.error("Error [create_keypair(shade_client)]. " + "Exception message, '%s'", o_exc.orig_message) + + +def create_instance_and_wait_for_active(shade_client, name, image, + flavor, auto_ip=True, ips=None, + ip_pool=None, root_volume=None, + terminate_volume=False, wait=True, + timeout=180, reuse_ips=True, + network=None, boot_from_volume=False, + volume_size='20', boot_volume=None, + volumes=None, nat_destination=None, + **kwargs): + """Create a virtual server instance. + + :param name:(string) Name of the server. + :param image:(dict) Image dict, name or ID to boot with. Image is required + unless boot_volume is given. + :param flavor:(dict) Flavor dict, name or ID to boot onto. + :param auto_ip: Whether to take actions to find a routable IP for + the server. + :param ips: List of IPs to attach to the server. + :param ip_pool:(string) Name of the network or floating IP pool to get an + address from. + :param root_volume:(string) Name or ID of a volume to boot from. + (defaults to None - deprecated, use boot_volume) + :param boot_volume:(string) Name or ID of a volume to boot from. + :param terminate_volume:(bool) If booting from a volume, whether it should + be deleted when the server is destroyed. + :param volumes:(optional) A list of volumes to attach to the server. + :param wait:(optional) Wait for the address to appear as assigned to the server. + :param timeout: Seconds to wait, defaults to 60. + :param reuse_ips:(bool)Whether to attempt to reuse pre-existing + floating ips should a floating IP be needed. + :param network:(dict) Network dict or name or ID to attach the server to. + Mutually exclusive with the nics parameter. Can also be be + a list of network names or IDs or network dicts. + :param boot_from_volume:(bool) Whether to boot from volume. 'boot_volume' + implies True, but boot_from_volume=True with + no boot_volume is valid and will create a + volume from the image and use that. + :param volume_size: When booting an image from volume, how big should + the created volume be? + :param nat_destination: Which network should a created floating IP + be attached to, if it's not possible to infer from + the cloud's configuration. + :param meta:(optional) A dict of arbitrary key/value metadata to store for + this server. Both keys and values must be <=255 characters. + :param reservation_id: A UUID for the set of servers being requested. + :param min_count:(optional extension) The minimum number of servers to + launch. + :param max_count:(optional extension) The maximum number of servers to + launch. + :param security_groups: A list of security group names. + :param userdata: User data to pass to be exposed by the metadata server + this can be a file type object as well or a string. + :param key_name:(optional extension) Name of previously created keypair to + inject into the instance. + :param availability_zone: Name of the availability zone for instance + placement. + :param block_device_mapping:(optional) A dict of block device mappings for + this server. + :param block_device_mapping_v2:(optional) A dict of block device mappings + for this server. + :param nics:(optional extension) An ordered list of nics to be added to + this server, with information about connected networks, fixed + IPs, port etc. + :param scheduler_hints:(optional extension) Arbitrary key-value pairs + specified by the client to help boot an instance. + :param config_drive:(optional extension) Value for config drive either + boolean, or volume-id. + :param disk_config:(optional extension) Control how the disk is partitioned + when the server is created. Possible values are 'AUTO' + or 'MANUAL'. + :param admin_pass:(optional extension) Add a user supplied admin password. + + :returns: The created server. + """ + try: + return shade_client.create_server( + name, image, flavor, auto_ip=auto_ip, ips=ips, ip_pool=ip_pool, + root_volume=root_volume, terminate_volume=terminate_volume, + wait=wait, timeout=timeout, reuse_ips=reuse_ips, network=network, + boot_from_volume=boot_from_volume, volume_size=volume_size, + boot_volume=boot_volume, volumes=volumes, + nat_destination=nat_destination, **kwargs) + except exc.OpenStackCloudException as o_exc: + log.error("Error [create_instance(shade_client)]. " + "Exception message, '%s'", o_exc.orig_message) + + +def attach_volume_to_server(shade_client, server_name_or_id, volume_name_or_id, + device=None, wait=True, timeout=None): + """Attach a volume to a server. + + This will attach a volume, described by the passed in volume + dict, to the server described by the passed in server dict on the named + device on the server. + + If the volume is already attached to the server, or generally not + available, then an exception is raised. To re-attach to a server, + but under a different device, the user must detach it first. + + :param server_name_or_id:(string) The server name or id to attach to. + :param volume_name_or_id:(string) The volume name or id to attach. + :param device:(string) The device name where the volume will attach. + :param wait:(bool) If true, waits for volume to be attached. + :param timeout: Seconds to wait for volume attachment. None is forever. + + :returns: True if attached successful, False otherwise. + """ + try: + server = shade_client.get_server(name_or_id=server_name_or_id) + volume = shade_client.get_volume(volume_name_or_id) + shade_client.attach_volume( + server, volume, device=device, wait=wait, timeout=timeout) return True - - -def delete_instance(nova_client, instance_id): # pragma: no cover - try: - nova_client.servers.force_delete(instance_id) - except Exception: - log.exception("Error [delete_instance(nova_client, '%s')]", - instance_id) + except exc.OpenStackCloudException as o_exc: + log.error("Error [attach_volume_to_server(shade_client)]. " + "Exception message: %s", o_exc.orig_message) return False - else: - return True -def remove_host_from_aggregate(nova_client, aggregate_name, - compute_host): # pragma: no cover - try: - aggregate_id = get_aggregate_id(nova_client, aggregate_name) - nova_client.aggregates.remove_host(aggregate_id, compute_host) - except Exception: - log.exception("Error remove_host_from_aggregate(nova_client, %s, %s)", - aggregate_name, compute_host) +def delete_instance(shade_client, name_or_id, wait=False, timeout=180, + delete_ips=False, delete_ip_retry=1): + """Delete a server instance. + + :param name_or_id: name or ID of the server to delete + :param wait:(bool) If true, waits for server to be deleted. + :param timeout:(int) Seconds to wait for server deletion. + :param delete_ips:(bool) If true, deletes any floating IPs associated with + the instance. + :param delete_ip_retry:(int) Number of times to retry deleting + any floating ips, should the first try be + unsuccessful. + :returns: True if delete succeeded, False otherwise. + """ + try: + return shade_client.delete_server( + name_or_id, wait=wait, timeout=timeout, delete_ips=delete_ips, + delete_ip_retry=delete_ip_retry) + except exc.OpenStackCloudException as o_exc: + log.error("Error [delete_instance(shade_client, '%s')]. " + "Exception message: %s", name_or_id, + o_exc.orig_message) return False - else: - return True - -def remove_hosts_from_aggregate(nova_client, - aggregate_name): # pragma: no cover - aggregate_id = get_aggregate_id(nova_client, aggregate_name) - hosts = nova_client.aggregates.get(aggregate_id).hosts - assert( - all(remove_host_from_aggregate(nova_client, aggregate_name, host) - for host in hosts)) +def get_server(shade_client, name_or_id=None, filters=None, detailed=False, + bare=False): + """Get a server by name or ID. -def delete_aggregate(nova_client, aggregate_name): # pragma: no cover - try: - remove_hosts_from_aggregate(nova_client, aggregate_name) - nova_client.aggregates.delete(aggregate_name) - except Exception: - log.exception("Error [delete_aggregate(nova_client, %s)]", - aggregate_name) - return False - else: - return True - + :param name_or_id: Name or ID of the server. + :param filters:(dict) A dictionary of meta data to use for further + filtering. + :param detailed:(bool) Whether or not to add detailed additional + information. + :param bare:(bool) Whether to skip adding any additional information to the + server record. -def get_server_by_name(name): # pragma: no cover + :returns: A server ``munch.Munch`` or None if no matching server is found. + """ try: - return get_nova_client().servers.list(search_opts={'name': name})[0] - except IndexError: - log.exception('Failed to get nova client') - raise + return shade_client.get_server(name_or_id=name_or_id, filters=filters, + detailed=detailed, bare=bare) + except exc.OpenStackCloudException as o_exc: + log.error("Error [get_server(shade_client, '%s')]. " + "Exception message: %s", name_or_id, o_exc.orig_message) def create_flavor(name, ram, vcpus, disk, **kwargs): # pragma: no cover try: - return get_nova_client().flavors.create(name, ram, vcpus, disk, **kwargs) - except Exception: + return get_nova_client().flavors.create(name, ram, vcpus, + disk, **kwargs) + except Exception: # pylint: disable=broad-except log.exception("Error [create_flavor(nova_client, %s, %s, %s, %s, %s)]", name, ram, disk, vcpus, kwargs['is_public']) return None -def get_image_by_name(name): # pragma: no cover - images = get_nova_client().images.list() - try: - return next((a for a in images if a.name == name)) - except StopIteration: - log.exception('No image matched') - - def get_flavor_id(nova_client, flavor_name): # pragma: no cover flavors = nova_client.flavors.list(detailed=True) flavor_id = '' @@ -408,405 +393,512 @@ def get_flavor_id(nova_client, flavor_name): # pragma: no cover return flavor_id -def get_flavor_by_name(name): # pragma: no cover - flavors = get_nova_client().flavors.list() - try: - return next((a for a in flavors if a.name == name)) - except StopIteration: - log.exception('No flavor matched') - - -def check_status(status, name, iterations, interval): # pragma: no cover - for i in range(iterations): - try: - server = get_server_by_name(name) - except IndexError: - log.error('Cannot found %s server', name) - raise +def get_flavor(shade_client, name_or_id, filters=None, get_extra=True): + """Get a flavor by name or ID. - if server.status == status: - return True + :param name_or_id: Name or ID of the flavor. + :param filters: A dictionary of meta data to use for further filtering. + :param get_extra: Whether or not the list_flavors call should get the extra + flavor specs. - time.sleep(interval) - return False + :returns: A flavor ``munch.Munch`` or None if no matching flavor is found. + """ + try: + return shade_client.get_flavor(name_or_id, filters=filters, + get_extra=get_extra) + except exc.OpenStackCloudException as o_exc: + log.error("Error [get_flavor(shade_client, '%s')]. " + "Exception message: %s", name_or_id, o_exc.orig_message) def delete_flavor(flavor_id): # pragma: no cover try: get_nova_client().flavors.delete(flavor_id) - except Exception: + except Exception: # pylint: disable=broad-except log.exception("Error [delete_flavor(nova_client, %s)]", flavor_id) return False else: return True -def delete_keypair(nova_client, key): # pragma: no cover +def delete_keypair(shade_client, name): + """Delete a keypair. + + :param name: Name of the keypair to delete. + + :returns: True if delete succeeded, False otherwise. + """ try: - nova_client.keypairs.delete(key=key) - return True - except Exception: - log.exception("Error [delete_keypair(nova_client)]") + return shade_client.delete_keypair(name) + except exc.OpenStackCloudException as o_exc: + log.error("Error [delete_neutron_router(shade_client, '%s')]. " + "Exception message: %s", name, o_exc.orig_message) return False # ********************************************* # NEUTRON # ********************************************* -def get_network_by_name(network_name): # pragma: no cover - try: - networks = get_neutron_client().list_networks()['networks'] - return next((n for n in networks if n['name'] == network_name), None) - except Exception: - log.exception("Error [get_instance_by_id(nova_client, '%s')]", - network_name) - - -def get_network_id(neutron_client, network_name): # pragma: no cover - networks = neutron_client.list_networks()['networks'] - return next((n['id'] for n in networks if n['name'] == network_name), None) - - -def get_port_id_by_ip(neutron_client, ip_address): # pragma: no cover - ports = neutron_client.list_ports()['ports'] - return next((i['id'] for i in ports for j in i.get( - 'fixed_ips') if j['ip_address'] == ip_address), None) - - -def create_neutron_net(neutron_client, json_body): # pragma: no cover - try: - network = neutron_client.create_network(body=json_body) - return network['network']['id'] - except Exception: - log.error("Error [create_neutron_net(neutron_client)]") - raise Exception("operation error") +def create_neutron_net(shade_client, network_name, shared=False, + admin_state_up=True, external=False, provider=None, + project_id=None): + """Create a neutron network. + + :param network_name:(string) name of the network being created. + :param shared:(bool) whether the network is shared. + :param admin_state_up:(bool) set the network administrative state. + :param external:(bool) whether this network is externally accessible. + :param provider:(dict) a dict of network provider options. + :param project_id:(string) specify the project ID this network + will be created on (admin-only). + :returns:(string) the network id. + """ + try: + networks = shade_client.create_network( + name=network_name, shared=shared, admin_state_up=admin_state_up, + external=external, provider=provider, project_id=project_id) + return networks['id'] + except exc.OpenStackCloudException as o_exc: + log.error("Error [create_neutron_net(shade_client)]." + "Exception message, '%s'", o_exc.orig_message) return None -def delete_neutron_net(neutron_client, network_id): # pragma: no cover +def delete_neutron_net(shade_client, network_id): try: - neutron_client.delete_network(network_id) - return True - except Exception: - log.error("Error [delete_neutron_net(neutron_client, '%s')]" % network_id) + return shade_client.delete_network(network_id) + except exc.OpenStackCloudException: + log.error("Error [delete_neutron_net(shade_client, '%s')]", network_id) return False -def create_neutron_subnet(neutron_client, json_body): # pragma: no cover - try: - subnet = neutron_client.create_subnet(body=json_body) - return subnet['subnets'][0]['id'] - except Exception: - log.error("Error [create_neutron_subnet") - raise Exception("operation error") +def create_neutron_subnet(shade_client, network_name_or_id, cidr=None, + ip_version=4, enable_dhcp=False, subnet_name=None, + tenant_id=None, allocation_pools=None, + gateway_ip=None, disable_gateway_ip=False, + dns_nameservers=None, host_routes=None, + ipv6_ra_mode=None, ipv6_address_mode=None, + use_default_subnetpool=False): + """Create a subnet on a specified network. + + :param network_name_or_id:(string) the unique name or ID of the + attached network. If a non-unique name is + supplied, an exception is raised. + :param cidr:(string) the CIDR. + :param ip_version:(int) the IP version. + :param enable_dhcp:(bool) whether DHCP is enable. + :param subnet_name:(string) the name of the subnet. + :param tenant_id:(string) the ID of the tenant who owns the network. + :param allocation_pools: A list of dictionaries of the start and end + addresses for the allocation pools. + :param gateway_ip:(string) the gateway IP address. + :param disable_gateway_ip:(bool) whether gateway IP address is enabled. + :param dns_nameservers: A list of DNS name servers for the subnet. + :param host_routes: A list of host route dictionaries for the subnet. + :param ipv6_ra_mode:(string) IPv6 Router Advertisement mode. + Valid values are: 'dhcpv6-stateful', + 'dhcpv6-stateless', or 'slaac'. + :param ipv6_address_mode:(string) IPv6 address mode. + Valid values are: 'dhcpv6-stateful', + 'dhcpv6-stateless', or 'slaac'. + :param use_default_subnetpool:(bool) use the default subnetpool for + ``ip_version`` to obtain a CIDR. It is + required to pass ``None`` to the ``cidr`` + argument when enabling this option. + :returns:(string) the subnet id. + """ + try: + subnet = shade_client.create_subnet( + network_name_or_id, cidr=cidr, ip_version=ip_version, + enable_dhcp=enable_dhcp, subnet_name=subnet_name, + tenant_id=tenant_id, allocation_pools=allocation_pools, + gateway_ip=gateway_ip, disable_gateway_ip=disable_gateway_ip, + dns_nameservers=dns_nameservers, host_routes=host_routes, + ipv6_ra_mode=ipv6_ra_mode, ipv6_address_mode=ipv6_address_mode, + use_default_subnetpool=use_default_subnetpool) + return subnet['id'] + except exc.OpenStackCloudException as o_exc: + log.error("Error [create_neutron_subnet(shade_client)]. " + "Exception message: %s", o_exc.orig_message) return None -def create_neutron_router(neutron_client, json_body): # pragma: no cover - try: - router = neutron_client.create_router(json_body) - return router['router']['id'] - except Exception: - log.error("Error [create_neutron_router(neutron_client)]") - raise Exception("operation error") - return None +def create_neutron_router(shade_client, name=None, admin_state_up=True, + ext_gateway_net_id=None, enable_snat=None, + ext_fixed_ips=None, project_id=None): + """Create a logical router. + :param name:(string) the router name. + :param admin_state_up:(bool) the administrative state of the router. + :param ext_gateway_net_id:(string) network ID for the external gateway. + :param enable_snat:(bool) enable Source NAT (SNAT) attribute. + :param ext_fixed_ips: List of dictionaries of desired IP and/or subnet + on the external network. + :param project_id:(string) project ID for the router. -def delete_neutron_router(neutron_client, router_id): # pragma: no cover + :returns:(string) the router id. + """ try: - neutron_client.delete_router(router=router_id) - return True - except Exception: - log.error("Error [delete_neutron_router(neutron_client, '%s')]" % router_id) - return False + router = shade_client.create_router( + name, admin_state_up, ext_gateway_net_id, enable_snat, + ext_fixed_ips, project_id) + return router['id'] + except exc.OpenStackCloudException as o_exc: + log.error("Error [create_neutron_router(shade_client)]. " + "Exception message: %s", o_exc.orig_message) -def remove_gateway_router(neutron_client, router_id): # pragma: no cover +def delete_neutron_router(shade_client, router_id): try: - neutron_client.remove_gateway_router(router_id) - return True - except Exception: - log.error("Error [remove_gateway_router(neutron_client, '%s')]" % router_id) + return shade_client.delete_router(router_id) + except exc.OpenStackCloudException as o_exc: + log.error("Error [delete_neutron_router(shade_client, '%s')]. " + "Exception message: %s", router_id, o_exc.orig_message) return False -def remove_interface_router(neutron_client, router_id, subnet_id, - **json_body): # pragma: no cover - json_body.update({"subnet_id": subnet_id}) +def remove_gateway_router(neutron_client, router_id): # pragma: no cover try: - neutron_client.remove_interface_router(router=router_id, - body=json_body) + neutron_client.remove_gateway_router(router_id) return True - except Exception: - log.error("Error [remove_interface_router(neutron_client, '%s', " - "'%s')]" % (router_id, subnet_id)) + except Exception: # pylint: disable=broad-except + log.error("Error [remove_gateway_router(neutron_client, '%s')]", + router_id) return False -def create_floating_ip(neutron_client, extnet_id): # pragma: no cover - props = {'floating_network_id': extnet_id} - try: - ip_json = neutron_client.create_floatingip({'floatingip': props}) - fip_addr = ip_json['floatingip']['floating_ip_address'] - fip_id = ip_json['floatingip']['id'] - except Exception: - log.error("Error [create_floating_ip(neutron_client)]") - return None - return {'fip_addr': fip_addr, 'fip_id': fip_id} +def remove_router_interface(shade_client, router, subnet_id=None, + port_id=None): + """Detach a subnet from an internal router interface. + At least one of subnet_id or port_id must be supplied. If you specify both + subnet and port ID, the subnet ID must correspond to the subnet ID of the + first IP address on the port specified by the port ID. + Otherwise an error occurs. -def delete_floating_ip(nova_client, floatingip_id): # pragma: no cover + :param router: The dict object of the router being changed + :param subnet_id:(string) The ID of the subnet to use for the interface + :param port_id:(string) The ID of the port to use for the interface + :returns: True on success + """ try: - nova_client.floating_ips.delete(floatingip_id) + shade_client.remove_router_interface( + router, subnet_id=subnet_id, port_id=port_id) return True - except Exception: - log.error("Error [delete_floating_ip(nova_client, '%s')]" % floatingip_id) + except exc.OpenStackCloudException as o_exc: + log.error("Error [remove_interface_router(shade_client)]. " + "Exception message: %s", o_exc.orig_message) return False -def get_security_groups(neutron_client): # pragma: no cover - try: - security_groups = neutron_client.list_security_groups()[ - 'security_groups'] - return security_groups - except Exception: - log.error("Error [get_security_groups(neutron_client)]") - return None - - -def get_security_group_id(neutron_client, sg_name): # pragma: no cover - security_groups = get_security_groups(neutron_client) - id = '' - for sg in security_groups: - if sg['name'] == sg_name: - id = sg['id'] - break - return id - - -def create_security_group(neutron_client, sg_name, sg_description): # pragma: no cover - json_body = {'security_group': {'name': sg_name, - 'description': sg_description}} - try: - secgroup = neutron_client.create_security_group(json_body) - return secgroup['security_group'] - except Exception: - log.error("Error [create_security_group(neutron_client, '%s', " - "'%s')]" % (sg_name, sg_description)) - return None +def create_floating_ip(shade_client, network_name_or_id=None, server=None, + fixed_address=None, nat_destination=None, + port=None, wait=False, timeout=60): + """Allocate a new floating IP from a network or a pool. + + :param network_name_or_id: Name or ID of the network + that the floating IP should come from. + :param server: Server dict for the server to create + the IP for and to which it should be attached. + :param fixed_address: Fixed IP to attach the floating ip to. + :param nat_destination: Name or ID of the network + that the fixed IP to attach the floating + IP to should be on. + :param port: The port ID that the floating IP should be + attached to. Specifying a port conflicts with specifying a + server,fixed_address or nat_destination. + :param wait: Whether to wait for the IP to be active.Only applies + if a server is provided. + :param timeout: How long to wait for the IP to be active.Only + applies if a server is provided. + + :returns:Floating IP id and address + """ + try: + fip = shade_client.create_floating_ip( + network=network_name_or_id, server=server, + fixed_address=fixed_address, nat_destination=nat_destination, + port=port, wait=wait, timeout=timeout) + return {'fip_addr': fip['floating_ip_address'], 'fip_id': fip['id']} + except exc.OpenStackCloudException as o_exc: + log.error("Error [create_floating_ip(shade_client)]. " + "Exception message: %s", o_exc.orig_message) + + +def delete_floating_ip(shade_client, floating_ip_id, retry=1): + try: + return shade_client.delete_floating_ip(floating_ip_id=floating_ip_id, + retry=retry) + except exc.OpenStackCloudException as o_exc: + log.error("Error [delete_floating_ip(shade_client,'%s')]. " + "Exception message: %s", floating_ip_id, o_exc.orig_message) + return False -def create_secgroup_rule(neutron_client, sg_id, direction, protocol, - port_range_min=None, port_range_max=None, - **json_body): # pragma: no cover - # We create a security group in 2 steps - # 1 - we check the format and set the json body accordingly - # 2 - we call neturon client to create the security group - - # Format check - json_body.update({'security_group_rule': {'direction': direction, - 'security_group_id': sg_id, 'protocol': protocol}}) - # parameters may be - # - both None => we do nothing - # - both Not None => we add them to the json description - # but one cannot be None is the other is not None - if (port_range_min is not None and port_range_max is not None): - # add port_range in json description - json_body['security_group_rule']['port_range_min'] = port_range_min - json_body['security_group_rule']['port_range_max'] = port_range_max - log.debug("Security_group format set (port range included)") - else: - # either both port range are set to None => do nothing - # or one is set but not the other => log it and return False - if port_range_min is None and port_range_max is None: - log.debug("Security_group format set (no port range mentioned)") - else: - log.error("Bad security group format." - "One of the port range is not properly set:" - "range min: {}," - "range max: {}".format(port_range_min, - port_range_max)) - return False - - # Create security group using neutron client - try: - neutron_client.create_security_group_rule(json_body) +def create_security_group_rule(shade_client, secgroup_name_or_id, + port_range_min=None, port_range_max=None, + protocol=None, remote_ip_prefix=None, + remote_group_id=None, direction='ingress', + ethertype='IPv4', project_id=None): + """Create a new security group rule + + :param secgroup_name_or_id:(string) The security group name or ID to + associate with this security group rule. If a + non-unique group name is given, an exception is + raised. + :param port_range_min:(int) The minimum port number in the range that is + matched by the security group rule. If the protocol + is TCP or UDP, this value must be less than or equal + to the port_range_max attribute value. If nova is + used by the cloud provider for security groups, then + a value of None will be transformed to -1. + :param port_range_max:(int) The maximum port number in the range that is + matched by the security group rule. The + port_range_min attribute constrains the + port_range_max attribute. If nova is used by the + cloud provider for security groups, then a value of + None will be transformed to -1. + :param protocol:(string) The protocol that is matched by the security group + rule. Valid values are None, tcp, udp, and icmp. + :param remote_ip_prefix:(string) The remote IP prefix to be associated with + this security group rule. This attribute matches + the specified IP prefix as the source IP address of + the IP packet. + :param remote_group_id:(string) The remote group ID to be associated with + this security group rule. + :param direction:(string) Ingress or egress: The direction in which the + security group rule is applied. + :param ethertype:(string) Must be IPv4 or IPv6, and addresses represented + in CIDR must match the ingress or egress rules. + :param project_id:(string) Specify the project ID this security group will + be created on (admin-only). + + :returns: True on success. + """ + + try: + shade_client.create_security_group_rule( + secgroup_name_or_id, port_range_min=port_range_min, + port_range_max=port_range_max, protocol=protocol, + remote_ip_prefix=remote_ip_prefix, remote_group_id=remote_group_id, + direction=direction, ethertype=ethertype, project_id=project_id) return True - except Exception: - log.exception("Impossible to create_security_group_rule," - "security group rule probably already exists") + except exc.OpenStackCloudException as op_exc: + log.error("Failed to create_security_group_rule(shade_client). " + "Exception message: %s", op_exc.orig_message) return False -def create_security_group_full(neutron_client, - sg_name, sg_description): # pragma: no cover - sg_id = get_security_group_id(neutron_client, sg_name) - if sg_id != '': - log.info("Using existing security group '%s'..." % sg_name) - else: - log.info("Creating security group '%s'..." % sg_name) - SECGROUP = create_security_group(neutron_client, - sg_name, - sg_description) - if not SECGROUP: - log.error("Failed to create the security group...") - return None - - sg_id = SECGROUP['id'] - - log.debug("Security group '%s' with ID=%s created successfully." - % (SECGROUP['name'], sg_id)) - - log.debug("Adding ICMP rules in security group '%s'..." - % sg_name) - if not create_secgroup_rule(neutron_client, sg_id, - 'ingress', 'icmp'): - log.error("Failed to create the security group rule...") - return None - - log.debug("Adding SSH rules in security group '%s'..." - % sg_name) - if not create_secgroup_rule( - neutron_client, sg_id, 'ingress', 'tcp', '22', '22'): - log.error("Failed to create the security group rule...") - return None - - if not create_secgroup_rule( - neutron_client, sg_id, 'egress', 'tcp', '22', '22'): - log.error("Failed to create the security group rule...") - return None - return sg_id +def create_security_group_full(shade_client, sg_name, + sg_description, project_id=None): + security_group = shade_client.get_security_group(sg_name) + + if security_group: + log.info("Using existing security group '%s'...", sg_name) + return security_group['id'] + + log.info("Creating security group '%s'...", sg_name) + try: + security_group = shade_client.create_security_group( + sg_name, sg_description, project_id=project_id) + except (exc.OpenStackCloudException, + exc.OpenStackCloudUnavailableFeature) as op_exc: + log.error("Error [create_security_group(shade_client, %s, %s)]. " + "Exception message: %s", sg_name, sg_description, + op_exc.orig_message) + return + + log.debug("Security group '%s' with ID=%s created successfully.", + security_group['name'], security_group['id']) + + log.debug("Adding ICMP rules in security group '%s'...", sg_name) + if not create_security_group_rule(shade_client, security_group['id'], + direction='ingress', protocol='icmp'): + log.error("Failed to create the security group rule...") + shade_client.delete_security_group(sg_name) + return + + log.debug("Adding SSH rules in security group '%s'...", sg_name) + if not create_security_group_rule(shade_client, security_group['id'], + direction='ingress', protocol='tcp', + port_range_min='22', + port_range_max='22'): + log.error("Failed to create the security group rule...") + shade_client.delete_security_group(sg_name) + return + + if not create_security_group_rule(shade_client, security_group['id'], + direction='egress', protocol='tcp', + port_range_min='22', + port_range_max='22'): + log.error("Failed to create the security group rule...") + shade_client.delete_security_group(sg_name) + return + return security_group['id'] # ********************************************* # GLANCE # ********************************************* -def get_image_id(glance_client, image_name): # pragma: no cover - images = glance_client.images.list() - return next((i.id for i in images if i.name == image_name), None) - - -def create_image(glance_client, image_name, file_path, disk_format, - container_format, min_disk, min_ram, protected, tag, - public, **kwargs): # pragma: no cover - if not os.path.isfile(file_path): - log.error("Error: file %s does not exist." % file_path) - return None - try: - image_id = get_image_id(glance_client, image_name) +def create_image(shade_client, name, filename=None, container='images', + md5=None, sha256=None, disk_format=None, + container_format=None, disable_vendor_agent=True, + wait=False, timeout=3600, allow_duplicates=False, meta=None, + volume=None, **kwargs): + """Upload an image. + + :param name:(str) Name of the image to create. If it is a pathname of an + image, the name will be constructed from the extensionless + basename of the path. + :param filename:(str) The path to the file to upload, if needed. + :param container:(str) Name of the container in swift where images should + be uploaded for import if the cloud requires such a thing. + :param md5:(str) md5 sum of the image file. If not given, an md5 will + be calculated. + :param sha256:(str) sha256 sum of the image file. If not given, an md5 + will be calculated. + :param disk_format:(str) The disk format the image is in. + :param container_format:(str) The container format the image is in. + :param disable_vendor_agent:(bool) Whether or not to append metadata + flags to the image to inform the cloud in + question to not expect a vendor agent to be running. + :param wait:(bool) If true, waits for image to be created. + :param timeout:(str) Seconds to wait for image creation. + :param allow_duplicates:(bool) If true, skips checks that enforce unique + image name. + :param meta:(dict) A dict of key/value pairs to use for metadata that + bypasses automatic type conversion. + :param volume:(str) Name or ID or volume object of a volume to create an + image from. + Additional kwargs will be passed to the image creation as additional + metadata for the image and will have all values converted to string + except for min_disk, min_ram, size and virtual_size which will be + converted to int. + If you are sure you have all of your data types correct or have an + advanced need to be explicit, use meta. If you are just a normal + consumer, using kwargs is likely the right choice. + If a value is in meta and kwargs, meta wins. + :returns: Image id + """ + try: + image_id = shade_client.get_image_id(name) if image_id is not None: - log.info("Image %s already exists." % image_name) - else: - log.info("Creating image '%s' from '%s'...", image_name, file_path) - - image = glance_client.images.create(name=image_name, - visibility=public, - disk_format=disk_format, - container_format=container_format, - min_disk=min_disk, - min_ram=min_ram, - tags=tag, - protected=protected, - **kwargs) - image_id = image.id - with open(file_path) as image_data: - glance_client.images.upload(image_id, image_data) + log.info("Image %s already exists.", name) + return image_id + log.info("Creating image '%s'", name) + image = shade_client.create_image( + name, filename=filename, container=container, md5=md5, sha256=sha256, + disk_format=disk_format, container_format=container_format, + disable_vendor_agent=disable_vendor_agent, wait=wait, timeout=timeout, + allow_duplicates=allow_duplicates, meta=meta, volume=volume, **kwargs) + image_id = image["id"] return image_id - except Exception: - log.error("Error [create_glance_image(glance_client, '%s', '%s', '%s')]", - image_name, file_path, public) - return None + except exc.OpenStackCloudException as op_exc: + log.error("Failed to create_image(shade_client). " + "Exception message: %s", op_exc.orig_message) -def delete_image(glance_client, image_id): # pragma: no cover +def delete_image(shade_client, name_or_id, wait=False, timeout=3600, + delete_objects=True): try: - glance_client.images.delete(image_id) + return shade_client.delete_image(name_or_id, wait=wait, + timeout=timeout, + delete_objects=delete_objects) - except Exception: - log.exception("Error [delete_flavor(glance_client, %s)]", image_id) + except exc.OpenStackCloudException as op_exc: + log.error("Failed to delete_image(shade_client). " + "Exception message: %s", op_exc.orig_message) + return False + + +def list_images(shade_client=None): + if shade_client is None: + shade_client = get_shade_client() + + try: + return shade_client.list_images() + except exc.OpenStackCloudException as o_exc: + log.error("Error [list_images(shade_client)]." + "Exception message, '%s'", o_exc.orig_message) return False - else: - return True # ********************************************* # CINDER # ********************************************* -def get_volume_id(volume_name): # pragma: no cover - volumes = get_cinder_client().volumes.list() - return next((v.id for v in volumes if v.name == volume_name), None) - - -def create_volume(cinder_client, volume_name, volume_size, - volume_image=False): # pragma: no cover - try: - if volume_image: - volume = cinder_client.volumes.create(name=volume_name, - size=volume_size, - imageRef=volume_image) - else: - volume = cinder_client.volumes.create(name=volume_name, - size=volume_size) - return volume - except Exception: - log.exception("Error [create_volume(cinder_client, %s)]", - (volume_name, volume_size)) - return None +def get_volume_id(shade_client, volume_name): + return shade_client.get_volume_id(volume_name) -def delete_volume(cinder_client, volume_id, forced=False): # pragma: no cover - try: - if forced: - try: - cinder_client.volumes.detach(volume_id) - except: - log.error(sys.exc_info()[0]) - cinder_client.volumes.force_delete(volume_id) - else: - while True: - volume = get_cinder_client().volumes.get(volume_id) - if volume.status.lower() == 'available': - break - cinder_client.volumes.delete(volume_id) - return True - except Exception: - log.exception("Error [delete_volume(cinder_client, '%s')]" % volume_id) - return False +def get_volume(shade_client, name_or_id, filters=None): + """Get a volume by name or ID. + :param name_or_id: Name or ID of the volume. + :param filters: A dictionary of meta data to use for further filtering. -def detach_volume(server_id, volume_id): # pragma: no cover - try: - get_nova_client().volumes.delete_server_volume(server_id, volume_id) - return True - except Exception: - log.exception("Error [detach_server_volume(nova_client, '%s', '%s')]", - server_id, volume_id) - return False -# ********************************************* -# HEAT -# ********************************************* + :returns: A volume ``munch.Munch`` or None if no matching volume is found. + """ + return shade_client.get_volume(name_or_id, filters=filters) + + +def create_volume(shade_client, size, wait=True, timeout=None, + image=None, **kwargs): + """Create a volume. + :param size: Size, in GB of the volume to create. + :param name: (optional) Name for the volume. + :param description: (optional) Name for the volume. + :param wait: If true, waits for volume to be created. + :param timeout: Seconds to wait for volume creation. None is forever. + :param image: (optional) Image name, ID or object from which to create + the volume. -def get_stack(heat_stack_id): # pragma: no cover + :returns: The created volume object. + + """ try: - client = get_heat_client() - return client.stacks.get(heat_stack_id) - except Exception as e: - log.exception("Error [get_stack(heat_stack_id)]", e) + return shade_client.create_volume(size, wait=wait, timeout=timeout, + image=image, **kwargs) + except (exc.OpenStackCloudException, exc.OpenStackCloudTimeout) as op_exc: + log.error("Failed to create_volume(shade_client). " + "Exception message: %s", op_exc.orig_message) + + +def delete_volume(shade_client, name_or_id=None, wait=True, timeout=None): + """Delete a volume. + :param name_or_id:(string) Name or unique ID of the volume. + :param wait:(bool) If true, waits for volume to be deleted. + :param timeout:(string) Seconds to wait for volume deletion. None is forever. -def get_stack_resources(heat_stack_id): # pragma: no cover + :return: True on success, False otherwise. + """ try: - client = get_heat_client() - return client.resources.list(heat_stack_id) - except Exception as e: - log.exception("Error [get_stack_resources(heat_stack_id)]: %s", e) + return shade_client.delete_volume(name_or_id=name_or_id, + wait=wait, timeout=timeout) + except (exc.OpenStackCloudException, exc.OpenStackCloudTimeout) as o_exc: + log.error("Error [delete_volume(shade_client,'%s')]. " + "Exception message: %s", name_or_id, o_exc.orig_message) + return False + +def detach_volume(shade_client, server_name_or_id, volume_name_or_id, + wait=True, timeout=None): + """Detach a volume from a server. -def get_stack_vms(heat_stack_id): # pragma: no cover - resources = get_stack_resources(heat_stack_id) - ret_vms = [] - for resource in resources: - if resource.resource_type == "OS::Nova::Server": - ret_vms.append(resource) - return ret_vms + :param server_name_or_id: The server name or id to detach from. + :param volume_name_or_id: The volume name or id to detach. + :param wait: If true, waits for volume to be detached. + :param timeout: Seconds to wait for volume detachment. None is forever. + + :return: True on success. + """ + try: + volume = shade_client.get_volume(volume_name_or_id) + server = get_server(shade_client, name_or_id=server_name_or_id) + shade_client.detach_volume(server, volume, wait=wait, timeout=timeout) + return True + except (exc.OpenStackCloudException, exc.OpenStackCloudTimeout) as o_exc: + log.error("Error [detach_volume(shade_client)]. " + "Exception message: %s", o_exc.orig_message) + return False diff --git a/vnftest/common/utils.py b/vnftest/common/utils.py index dfd32d5..b3f0c05 100644 --- a/vnftest/common/utils.py +++ b/vnftest/common/utils.py @@ -204,16 +204,6 @@ def result_handler(status, data): return jsonify(result) -def change_obj_to_dict(obj): - dic = {} - for k, v in vars(obj).items(): - try: - vars(v) - except TypeError: - dic.update({k: v}) - return dic - - def set_dict_value(dic, keys, value): return_dic = dic @@ -413,9 +403,26 @@ class dotdict(dict): __delattr__ = dict.__delitem__ +def deep_dotdict(obj): + if isinstance(obj, dict): + dot_dict = {} + for k, v in obj.items(): + if isinstance(k, basestring) and not k.startswith('_'): + v = deep_dotdict(v) + dot_dict[k] = v + return dotdict(dot_dict) + if isinstance(obj, list): + new_list = [] + for element in obj: + element = deep_dotdict(element) + new_list.append(element) + return new_list + return obj + + def normalize_data_struct(obj): - if isinstance(obj, basestring): - return [obj] + if obj is None: + return None if isinstance(obj, list): nomalized_list = [] for element in obj: @@ -424,11 +431,15 @@ def normalize_data_struct(obj): return nomalized_list if isinstance(obj, dict): normalized_dict = {} - for k, v in obj: - v = normalize_data_struct(v) - normalized_dict[k] = v + for k, v in obj.items(): + if isinstance(k, basestring) and not k.startswith('_'): + v = normalize_data_struct(v) + normalized_dict[k] = v return normalized_dict - return change_obj_to_dict(obj) + # return obj if it is string, integer, bool ect. + if not hasattr(obj, '__dict__'): + return obj + return normalize_data_struct(obj.__dict__) def xml_to_dict(xml_str): @@ -510,7 +521,6 @@ def format(in_obj, params): if not isinstance(in_obj, basestring): return in_obj - dotdict(params) ret_str = "" ret_obj = None for literal_text, field_name, format_spec, conversion in \ @@ -522,7 +532,7 @@ def format(in_obj, params): try: value = tmp_dict[field_name] except KeyError: - tmp_dict = dotdict(tmp_dict) + tmp_dict = deep_dotdict(tmp_dict) field_name = '{' + field_name + '}' value = field_name.format(**tmp_dict) if isinstance(value, basestring): diff --git a/vnftest/contexts/base.py b/vnftest/contexts/base.py index 29e3a19..c10732a 100644 --- a/vnftest/contexts/base.py +++ b/vnftest/contexts/base.py @@ -13,27 +13,69 @@ ############################################################################## import abc import six -from vnftest.common import openstack_utils import vnftest.common.utils as utils -import yaml import logging + +from vnftest.common import constants + LOG = logging.getLogger(__name__) +class Flags(object): + """Class to represent the status of the flags in a context""" + + _FLAGS = {'no_setup': False, + 'no_teardown': False, + 'os_cloud_config': constants.OS_CLOUD_DEFAULT_CONFIG} + + def __init__(self, **kwargs): + for name, value in self._FLAGS.items(): + setattr(self, name, value) + + for name, value in ((name, value) for (name, value) in kwargs.items() + if name in self._FLAGS): + setattr(self, name, value) + + def parse(self, **kwargs): + """Read in values matching the flags stored in this object""" + if not kwargs: + return + + for name, value in ((name, value) for (name, value) in kwargs.items() + if name in self._FLAGS): + setattr(self, name, value) + + @six.add_metaclass(abc.ABCMeta) class Context(object): """Class that represents a context in the logical model""" - list = [] + _list = [] def __init__(self): - Context.list.append(self) + Context._list.append(self) + self._flags = Flags() self._task_id = None self._name = None + self._name_task_id = None def init(self, attrs): self._task_id = attrs['task_id'] self._name = attrs['name'] + self._flags.parse(**attrs.get('flags', {})) + self._name_task_id = '{}-{}'.format( + self._name, self._task_id[:8]) + + @property + def name(self): + if self._flags.no_setup or self._flags.no_teardown: + return self._name + else: + return self._name_task_id + + @property + def assigned_name(self): + return self._name @staticmethod def get_cls(context_type): @@ -50,7 +92,7 @@ class Context(object): return Context.get_cls(context_type)() def _delete_context(self): - Context.list.remove(self) + Context._list.remove(self) @abc.abstractmethod def deploy(self): diff --git a/vnftest/contexts/heat.py b/vnftest/contexts/heat.py new file mode 100644 index 0000000..f6ca611 --- /dev/null +++ b/vnftest/contexts/heat.py @@ -0,0 +1,591 @@ +############################################################################## +# Copyright 2018 EuropeanSoftwareMarketingLtd. +# =================================================================== +# Licensed under the ApacheLicense, Version2.0 (the"License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under +# the License +############################################################################## +# vnftest comment: this is a modified copy of +# yardstick/benchmark/contexts/heat.py + +import collections +import logging +import os +import errno +from collections import OrderedDict + +import ipaddress +import pkg_resources + +from vnftest.contexts.base import Context +from vnftest.contexts.model import Network +from vnftest.contexts.model import PlacementGroup, ServerGroup +from vnftest.contexts.model import Server +from vnftest.contexts.model import update_scheduler_hints +from vnftest.common import exceptions as y_exc +from vnftest.common.openstack_utils import get_shade_client +from vnftest.orchestrator.heat import HeatStack +from vnftest.orchestrator.heat import HeatTemplate +from vnftest.common import constants as consts +from vnftest.common import utils +from vnftest.common.utils import source_env +from vnftest.common import openstack_utils + +LOG = logging.getLogger(__name__) + +DEFAULT_HEAT_TIMEOUT = 3600 + + +def join_args(sep, *args): + return sep.join(args) + + +def h_join(*args): + return '-'.join(args) + + +class HeatContext(Context): + """Class that represents a context in the logical model""" + + __context_type__ = "Heat" + + def __init__(self): + self.stack = None + self.networks = OrderedDict() + self.heat_timeout = None + self.servers = {} + self.placement_groups = [] + self.server_groups = [] + self.keypair_name = None + self.secgroup_name = None + self.security_group = None + self.attrs = {} + self._image = None + self._flavor = None + self.flavors = set() + self._user = None + self.template_file = None + self.heat_parameters = None + self.heat_timeout = None + self.key_filename = None + self._shade_client = None + self._operator_client = None + self.nodes = [] + self.controllers = [] + self.computes = [] + self.baremetals = [] + super(HeatContext, self).__init__() + + @staticmethod + def assign_external_network(networks): + if networks: + sorted_networks = sorted(networks.items()) + external_network = os.environ.get("EXTERNAL_NETWORK", "net04_ext") + + have_external_network = any(net.get("external_network") for net in networks.values()) + if not have_external_network: + # try looking for mgmt network first + try: + networks['mgmt']["external_network"] = external_network + except KeyError: + if sorted_networks: + # otherwise assign it to first network using os.environ + sorted_networks[0][1]["external_network"] = external_network + + return sorted_networks + return networks + + def init(self, attrs): + """Initializes itself from the supplied arguments""" + super(HeatContext, self).init(attrs) + + self.check_environment() + self._user = attrs.get("user") + + self.template_file = attrs.get("heat_template") + + self._shade_client = openstack_utils.get_shade_client() + self._operator_client = openstack_utils.get_shade_operator_client() + + self.heat_timeout = attrs.get("timeout", DEFAULT_HEAT_TIMEOUT) + if self.template_file: + self.heat_parameters = attrs.get("heat_parameters") + return + + self.keypair_name = None # h_join(self.name, "key") + + self.secgroup_name = None # h_join(self.name, "secgroup") + + self.security_group = attrs.get("security_group") + + self._image = attrs.get("image") + + self._flavor = attrs.get("flavor") + + self.placement_groups = [PlacementGroup(name, self, pg_attrs["policy"]) + for name, pg_attrs in attrs.get( + "placement_groups", {}).items()] + + self.server_groups = [ServerGroup(name, self, sg_attrs["policy"]) + for name, sg_attrs in attrs.get( + "server_groups", {}).items()] + + # we have to do this first, because we are injecting external_network + # into the dict + networks = attrs.get("networks", {}) + sorted_networks = self.assign_external_network(networks) + + if sorted_networks: + self.networks = OrderedDict( + (name, Network(name, self, net_attrs)) for name, net_attrs in + sorted_networks) + + for name, server_attrs in sorted(attrs["servers"].items()): + server = Server(name, self, server_attrs) + self.servers[name] = server + + self.attrs = attrs + + def check_environment(self): + try: + os.environ['OS_AUTH_URL'] + except KeyError: + try: + source_env(consts.OPENRC) + except IOError as e: + if e.errno != errno.EEXIST: + LOG.error('OPENRC file not found') + raise + else: + LOG.error('OS_AUTH_URL not found') + + @property + def image(self): + """returns application's default image name""" + return self._image + + @property + def flavor(self): + """returns application's default flavor name""" + return self._flavor + + @property + def user(self): + """return login user name corresponding to image""" + return self._user + + def _add_resources_to_template(self, template): + """add to the template the resources represented by this context""" + + if self.flavor: + if isinstance(self.flavor, dict): + flavor = self.flavor.setdefault("name", self.name + "-flavor") + template.add_flavor(**self.flavor) + self.flavors.add(flavor) + + # template.add_keypair(self.keypair_name, self.name) + # template.add_security_group(self.secgroup_name, self.security_group) + + for network in self.networks.values(): + # Using existing network + if network.is_existing(): + continue + template.add_network(network.stack_name, + network.physical_network, + network.provider, + network.segmentation_id, + network.port_security_enabled, + network.network_type) + template.add_subnet(network.subnet_stack_name, network.stack_name, + network.subnet_cidr, + network.enable_dhcp, + network.gateway_ip) + + if network.router: + template.add_router(network.router.stack_name, + network.router.external_gateway_info, + network.subnet_stack_name) + template.add_router_interface(network.router.stack_if_name, + network.router.stack_name, + network.subnet_stack_name) + + # create a list of servers sorted by increasing no of placement groups + list_of_servers = sorted(self.servers.itervalues(), + key=lambda s: len(s.placement_groups)) + + # + # add servers with scheduler hints derived from placement groups + # + + # create list of servers with availability policy + availability_servers = [] + for server in list_of_servers: + for pg in server.placement_groups: + if pg.policy == "availability": + availability_servers.append(server) + break + + for server in availability_servers: + if isinstance(server.flavor, dict): + try: + self.flavors.add(server.flavor["name"]) + except KeyError: + self.flavors.add(h_join(server.stack_name, "flavor")) + + # add servers with availability policy + added_servers = [] + for server in availability_servers: + scheduler_hints = {} + for pg in server.placement_groups: + update_scheduler_hints(scheduler_hints, added_servers, pg) + # workaround for openstack nova bug, check JIRA: vnftest-200 + # for details + if len(availability_servers) == 2: + if not scheduler_hints["different_host"]: + scheduler_hints.pop("different_host", None) + server.add_to_template(template, + list(self.networks.values()), + scheduler_hints) + else: + scheduler_hints["different_host"] = \ + scheduler_hints["different_host"][0] + server.add_to_template(template, + list(self.networks.values()), + scheduler_hints) + else: + server.add_to_template(template, + list(self.networks.values()), + scheduler_hints) + added_servers.append(server.stack_name) + + # create list of servers with affinity policy + affinity_servers = [] + for server in list_of_servers: + for pg in server.placement_groups: + if pg.policy == "affinity": + affinity_servers.append(server) + break + + # add servers with affinity policy + for server in affinity_servers: + if server.stack_name in added_servers: + continue + scheduler_hints = {} + for pg in server.placement_groups: + update_scheduler_hints(scheduler_hints, added_servers, pg) + server.add_to_template(template, list(self.networks.values()), + scheduler_hints) + added_servers.append(server.stack_name) + + # add server group + for sg in self.server_groups: + template.add_server_group(sg.name, sg.policy) + + # add remaining servers with no placement group configured + for server in list_of_servers: + # TODO placement_group and server_group should combine + if not server.placement_groups: + scheduler_hints = {} + # affinity/anti-aff server group + sg = server.server_group + if sg: + scheduler_hints["group"] = {'get_resource': sg.name} + server.add_to_template(template, + list(self.networks.values()), + scheduler_hints) + + def get_neutron_info(self): + if not self._shade_client: + self._shade_client = get_shade_client() + + networks = self._shade_client.list_networks() + for network in self.networks.values(): + for neutron_net in (net for net in networks if net.name == network.stack_name): + network.segmentation_id = neutron_net.get('provider:segmentation_id') + # we already have physical_network + # network.physical_network = neutron_net.get('provider:physical_network') + network.network_type = neutron_net.get('provider:network_type') + network.neutron_info = neutron_net + + def _create_new_stack(self, heat_template): + try: + return heat_template.create(block=True, + timeout=self.heat_timeout) + except KeyboardInterrupt: + raise y_exc.StackCreationInterrupt + except Exception: + LOG.exception("stack failed") + # let the other failures happen, we want stack trace + raise + + def _retrieve_existing_stack(self, stack_name): + stack = HeatStack(stack_name) + if stack.get(): + return stack + else: + LOG.warning("Stack %s does not exist", self.name) + return None + + def deploy(self): + """deploys template into a stack using cloud""" + LOG.info("Deploying context '%s' START", self.name) + + # self.key_filename = ''.join( + # [consts.VNFTEST_ROOT_PATH, + # '/vnftest/resources/files/vnftest_key-', + # self.name]) + # Permissions may have changed since creation; this can be fixed. If we + # overwrite the file, we lose future access to VMs using this key. + # As long as the file exists, even if it is unreadable, keep it intact + # if not os.path.exists(self.key_filename): + # SSH.gen_keys(self.key_filename) + + heat_template = HeatTemplate( + self.name, template_file=self.template_file, + heat_parameters=self.heat_parameters, + os_cloud_config=self._flags.os_cloud_config) + + if self.template_file is None: + self._add_resources_to_template(heat_template) + + if self._flags.no_setup: + # Try to get an existing stack, returns a stack or None + self.stack = self._retrieve_existing_stack(self.name) + if not self.stack: + self.stack = self._create_new_stack(heat_template) + + else: + self.stack = self._create_new_stack(heat_template) + + # TODO: use Neutron to get segmentation-id + self.get_neutron_info() + + # copy some vital stack output into server objects + for server in self.servers.itervalues(): + if server.networks: + self.update_networks(server) + if server.ports: + self.add_server_port(server) + + if server.floating_ip: + server.public_ip = \ + self.stack.outputs[server.floating_ip["stack_name"]] + + LOG.info("Deploying context '%s' DONE", self.name) + + @staticmethod + def _port_net_is_existing(port_info): + net_flags = port_info.get('net_flags', {}) + return net_flags.get(consts.IS_EXISTING) + + @staticmethod + def _port_net_is_public(port_info): + net_flags = port_info.get('net_flags', {}) + return net_flags.get(consts.IS_PUBLIC) + + def update_networks(self, server): + for network in server.networks: + if 'network' in network: + network_name = network['network'] + output_key = server.stack_name + ".networks.%s-ip" % network_name + ip = self.stack.outputs[output_key] + network["ip"] = ip + + def add_server_port(self, server): + server_ports = server.ports.values() + for server_port in server_ports: + port_info = server_port[0] + port_ip = self.stack.outputs[port_info["stack_name"]] + port_net_is_existing = self._port_net_is_existing(port_info) + port_net_is_public = self._port_net_is_public(port_info) + if port_net_is_existing and (port_net_is_public or + len(server_ports) == 1): + server.public_ip = port_ip + if not server.private_ip or len(server_ports) == 1: + server.private_ip = port_ip + + server.interfaces = {} + for network_name, ports in server.ports.items(): + for port in ports: + # port['port'] is either port name from mapping or default network_name + if self._port_net_is_existing(port): + continue + server.interfaces[port['port']] = self.make_interface_dict(network_name, + port['port'], + port['stack_name'], + self.stack.outputs) + server.override_ip(network_name, port) + + def make_interface_dict(self, network_name, port, stack_name, outputs): + private_ip = outputs[stack_name] + mac_address = outputs[h_join(stack_name, "mac_address")] + # these are attributes of the network, not the port + output_subnet_cidr = outputs[h_join(self.name, network_name, + 'subnet', 'cidr')] + + # these are attributes of the network, not the port + output_subnet_gateway = outputs[h_join(self.name, network_name, + 'subnet', 'gateway_ip')] + + return { + # add default port name + "name": port, + "private_ip": private_ip, + "subnet_id": outputs[h_join(stack_name, "subnet_id")], + "subnet_cidr": output_subnet_cidr, + "network": str(ipaddress.ip_network(output_subnet_cidr).network_address), + "netmask": str(ipaddress.ip_network(output_subnet_cidr).netmask), + "gateway_ip": output_subnet_gateway, + "mac_address": mac_address, + "device_id": outputs[h_join(stack_name, "device_id")], + "network_id": outputs[h_join(stack_name, "network_id")], + # this should be == vld_id for NSB tests + "network_name": network_name, + # to match vnf_generic + "local_mac": mac_address, + "local_ip": private_ip, + } + + def _delete_key_file(self): + if self.key_filename is not None: + try: + utils.remove_file(self.key_filename) + utils.remove_file(self.key_filename + ".pub") + except OSError: + LOG.exception("There was an error removing the key file %s", + self.key_filename) + + def undeploy(self): + """undeploys stack from cloud""" + if self._flags.no_teardown: + LOG.info("Undeploying context '%s' SKIP", self.name) + return + + if self.stack: + LOG.info("Undeploying context '%s' START", self.name) + self.stack.delete() + self.stack = None + LOG.info("Undeploying context '%s' DONE", self.name) + + self._delete_key_file() + + super(HeatContext, self).undeploy() + + @staticmethod + def generate_routing_table(server): + routes = [ + { + "network": intf["network"], + "netmask": intf["netmask"], + "if": name, + # We have to encode a None gateway as '' for Jinja2 to YAML conversion + "gateway": intf["gateway_ip"] if intf["gateway_ip"] else '', + } + for name, intf in server.interfaces.items() + ] + return routes + + def _get_server(self, attr_name): + """lookup server info by name from context + attr_name: either a name for a server created by vnftest or a dict + with attribute name mapping when using external heat templates + """ + if isinstance(attr_name, collections.Mapping): + node_name, cname = self.split_host_name(attr_name['name']) + if cname is None or cname != self.name: + return None + + # Create a dummy server instance for holding the *_ip attributes + server = Server(node_name, self, {}) + server.public_ip = self.stack.outputs.get( + attr_name.get("public_ip_attr", object()), None) + + server.private_ip = self.stack.outputs.get( + attr_name.get("private_ip_attr", object()), None) + else: + try: + server = self.servers[attr_name] + except KeyError: + attr_name_no_suffix = attr_name.split("-")[0] + server = self.servers.get(attr_name_no_suffix, None) + if server is None: + return None + + pkey = pkg_resources.resource_string( + 'vnftest.resources', + h_join('files/vnftest_key', self.name)).decode('utf-8') + key_filename = pkg_resources.resource_filename('vnftest.resources', + h_join('files/vnftest_key', self.name)) + result = { + "user": server._context.user, + "pkey": pkey, + "key_filename": key_filename, + "private_ip": server.private_ip, + "interfaces": server.interfaces, + "routing_table": self.generate_routing_table(server), + # empty IPv6 routing table + "nd_route_tbl": [], + # we want to save the contex name so we can generate pod.yaml + "name": server.name, + } + # Target server may only have private_ip + if server.public_ip: + result["ip"] = server.public_ip + + return result + + def _get_network(self, attr_name): + if not isinstance(attr_name, collections.Mapping): + network = self.networks.get(attr_name, None) + + else: + # Only take the first key, value + key, value = next(iter(attr_name.items()), (None, None)) + if key is None: + return None + network_iter = (n for n in self.networks.values() if getattr(n, key) == value) + network = next(network_iter, None) + + if network is None: + return None + + result = { + "name": network.name, + "segmentation_id": network.segmentation_id, + "network_type": network.network_type, + "physical_network": network.physical_network, + } + return result + + def _get_physical_nodes(self): + return self.nodes + + def _get_physical_node_for_server(self, server_name): + node_name, ctx_name = self.split_host_name(server_name) + if ctx_name is None or self.name != ctx_name: + return None + + matching_nodes = [s for s in self.servers.itervalues() if s.name == node_name] + if len(matching_nodes) == 0: + return None + + server = openstack_utils.get_server(self._shade_client, + name_or_id=server_name) + + if server: + server = server.toDict() + list_hypervisors = self._operator_client.list_hypervisors() + + for hypervisor in list_hypervisors: + if hypervisor.hypervisor_hostname == server['OS-EXT-SRV-ATTR:hypervisor_hostname']: + for node in self.nodes: + if node['ip'] == hypervisor.host_ip: + return "{}.{}".format(node['name'], self.name) + + return None diff --git a/vnftest/contexts/model.py b/vnftest/contexts/model.py new file mode 100644 index 0000000..52b8d34 --- /dev/null +++ b/vnftest/contexts/model.py @@ -0,0 +1,433 @@ +############################################################################## +# Copyright 2018 EuropeanSoftwareMarketingLtd. +# =================================================================== +# Licensed under the ApacheLicense, Version2.0 (the"License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under +# the License +############################################################################## +# vnftest comment: this is a modified copy of +# yardstick/benchmark/contexts/model.py +""" Logical model + +""" +from __future__ import absolute_import + +import six +import logging + +from collections import Mapping +from six.moves import range + +from vnftest.common import constants as consts + + +LOG = logging.getLogger(__name__) + + +class Object(object): + """Base class for classes in the logical model + Contains common attributes and methods + """ + + def __init__(self, name, context): + # model identities and reference + self.name = name + self._context = context + + # stack identities + self.stack_name = None + self.stack_id = None + + @property + def dn(self): + """returns distinguished name for object""" + return self.name + "." + self._context.name + + +class PlacementGroup(Object): + """Class that represents a placement group in the logical model + Concept comes from the OVF specification. Policy should be one of + "availability" or "affinity (there are more but they are not supported)" + """ + map = {} + + def __init__(self, name, context, policy): + if policy not in ["affinity", "availability"]: + raise ValueError("placement group '%s', policy '%s' is not valid" % + (name, policy)) + self.name = name + self.members = set() + self.stack_name = context.name + "-" + name + self.policy = policy + PlacementGroup.map[name] = self + + def add_member(self, name): + self.members.add(name) + + @staticmethod + def get(name): + return PlacementGroup.map.get(name) + + +class ServerGroup(Object): # pragma: no cover + """Class that represents a server group in the logical model + Policy should be one of "anti-affinity" or "affinity" + """ + map = {} + + def __init__(self, name, context, policy): + super(ServerGroup, self).__init__(name, context) + if policy not in {"affinity", "anti-affinity"}: + raise ValueError("server group '%s', policy '%s' is not valid" % + (name, policy)) + self.name = name + self.members = set() + self.stack_name = context.name + "-" + name + self.policy = policy + ServerGroup.map[name] = self + + def add_member(self, name): + self.members.add(name) + + @staticmethod + def get(name): + return ServerGroup.map.get(name) + + +class Router(Object): + """Class that represents a router in the logical model""" + + def __init__(self, name, network_name, context, external_gateway_info): + super(Router, self).__init__(name, context) + + self.stack_name = context.name + "-" + network_name + "-" + self.name + self.stack_if_name = self.stack_name + "-if0" + self.external_gateway_info = external_gateway_info + + +class Network(Object): + """Class that represents a network in the logical model""" + list = [] + + def __init__(self, name, context, attrs): + super(Network, self).__init__(name, context) + self.stack_name = context.name + "-" + self.name + self.subnet_stack_name = self.stack_name + "-subnet" + self.subnet_cidr = attrs.get('cidr', '10.0.1.0/24') + self.enable_dhcp = attrs.get('enable_dhcp', 'true') + self.router = None + self.physical_network = attrs.get('physical_network', 'physnet1') + self.provider = attrs.get('provider') + self.segmentation_id = attrs.get('segmentation_id') + self.network_type = attrs.get('network_type') + self.port_security_enabled = attrs.get('port_security_enabled') + self.vnic_type = attrs.get('vnic_type', 'normal') + self.allowed_address_pairs = attrs.get('allowed_address_pairs', []) + try: + # we require 'null' or '' to disable setting gateway_ip + self.gateway_ip = attrs['gateway_ip'] + except KeyError: + # default to explicit None + self.gateway_ip = None + else: + # null is None in YAML, so we have to convert back to string + if self.gateway_ip is None: + self.gateway_ip = "null" + + self.net_flags = attrs.get('net_flags', {}) + if self.is_existing(): + self.subnet = attrs.get('subnet') + if not self.subnet: + raise Warning('No subnet set in existing netwrok!') + else: + if "external_network" in attrs: + self.router = Router("router", self.name, + context, attrs["external_network"]) + Network.list.append(self) + + def is_existing(self): + net_is_existing = self.net_flags.get(consts.IS_EXISTING) + if net_is_existing and not isinstance(net_is_existing, bool): + raise SyntaxError('Network flags should be bool type!') + return net_is_existing + + def is_public(self): + net_is_public = self.net_flags.get(consts.IS_PUBLIC) + if net_is_public and not isinstance(net_is_public, bool): + raise SyntaxError('Network flags should be bool type!') + return net_is_public + + def has_route_to(self, network_name): + """determines if this network has a route to the named network""" + if self.router and self.router.external_gateway_info == network_name: + return True + return False + + @staticmethod + def find_by_route_to(external_network): + """finds a network that has a route to the specified network""" + for network in Network.list: + if network.has_route_to(external_network): + return network + + @staticmethod + def find_external_network(): + """return the name of an external network some network in this + context has a route to + """ + for network in Network.list: + if network.router: + return network.router.external_gateway_info + return None + + +class Server(Object): # pragma: no cover + """Class that represents a server in the logical model""" + list = [] + + def __init__(self, name, context, attrs): + super(Server, self).__init__(name, context) + self.stack_name = self.name + "." + context.name + self.keypair_name = context.keypair_name + self.secgroup_name = context.secgroup_name + self.user = context.user + self._context = context + self.public_ip = None + self.private_ip = None + self.user_data = '' + self.interfaces = {} + self.networks = None + self.ports = {} + self.floating_ip = {} + + if attrs is None: + attrs = {} + + self.placement_groups = [] + placement = attrs.get("placement", []) + placement = placement if isinstance(placement, list) else [placement] + for p in placement: + pg = PlacementGroup.get(p) + if not pg: + raise ValueError("server '%s', placement '%s' is invalid" % + (name, p)) + self.placement_groups.append(pg) + pg.add_member(self.stack_name) + + self.volume = None + if "volume" in attrs: + self.volume = attrs.get("volume") + + self.volume_mountpoint = None + if "volume_mountpoint" in attrs: + self.volume_mountpoint = attrs.get("volume_mountpoint") + + # support servergroup attr + self.server_group = None + sg = attrs.get("server_group") + if sg: + server_group = ServerGroup.get(sg) + if not server_group: + raise ValueError("server '%s', server_group '%s' is invalid" % + (name, sg)) + self.server_group = server_group + server_group.add_member(self.stack_name) + + self.instances = 1 + if "instances" in attrs: + self.instances = attrs["instances"] + + if "networks" in attrs: + self.networks = attrs["networks"] + else: + # dict with key network name, each item is a dict with port name and ip + self.network_ports = attrs.get("network_ports", {}) + + self.floating_ip = None + self.floating_ip_assoc = None + if "floating_ip" in attrs: + self.floating_ip_assoc = {} + + if self.floating_ip is not None: + ext_net = Network.find_external_network() + assert ext_net is not None + self.floating_ip["external_network"] = ext_net + + self._image = None + if "image" in attrs: + self._image = attrs["image"] + + self._flavor = None + if "flavor" in attrs: + self._flavor = attrs["flavor"] + + self.user_data = attrs.get('user_data', '') + self.availability_zone = attrs.get('availability_zone') + + Server.list.append(self) + + def override_ip(self, network_name, port): + def find_port_overrides(): + for p in ports: + # p can be string or dict + # we can't just use p[port['port'] in case p is a string + # and port['port'] is an int? + if isinstance(p, Mapping): + g = p.get(port['port']) + # filter out empty dicts + if g: + yield g + + ports = self.network_ports.get(network_name, []) + intf = self.interfaces[port['port']] + for override in find_port_overrides(): + intf['local_ip'] = override.get('local_ip', intf['local_ip']) + intf['netmask'] = override.get('netmask', intf['netmask']) + # only use the first value + break + + @property + def image(self): + """returns a server's image name""" + if self._image: + return self._image + else: + return self._context.image + + @property + def flavor(self): + """returns a server's flavor name""" + if self._flavor: + return self._flavor + else: + return self._context.flavor + + def _add_instance(self, template, server_name, networks, scheduler_hints): + """adds to the template one server and corresponding resources""" + port_name_list = None + if self.networks is None: + port_name_list = [] + for network in networks: + # if explicit mapping skip unused networks + if self.network_ports: + try: + ports = self.network_ports[network.name] + except KeyError: + # no port for this network + continue + else: + if isinstance(ports, six.string_types): + # because strings are iterable we have to check specifically + raise SyntaxError("network_port must be a list '{}'".format(ports)) + # convert port subdicts into their just port name + # port subdicts are used to override Heat IP address, + # but we just need the port name + # we allow duplicates here and let Heat raise the error + ports = [next(iter(p)) if isinstance(p, dict) else p for p in ports] + # otherwise add a port for every network with port name as network name + else: + ports = [network.name] + net_flags = network.net_flags + for port in ports: + port_name = "{0}-{1}-port".format(server_name, port) + port_info = {"stack_name": port_name, "port": port} + if net_flags: + port_info['net_flags'] = net_flags + self.ports.setdefault(network.name, []).append(port_info) + # we can't use secgroups if port_security_enabled is False + if network.port_security_enabled is False: + sec_group_id = None + else: + # if port_security_enabled is None we still need to add to secgroup + sec_group_id = self.secgroup_name + # don't refactor to pass in network object, that causes JSON + # circular ref encode errors + template.add_port(port_name, network, + sec_group_id=sec_group_id, + provider=network.provider, + allowed_address_pairs=network.allowed_address_pairs) + if network.is_public(): + port_name_list.insert(0, port_name) + else: + port_name_list.append(port_name) + + if self.floating_ip: + external_network = self.floating_ip["external_network"] + if network.has_route_to(external_network): + self.floating_ip["stack_name"] = server_name + "-fip" + template.add_floating_ip(self.floating_ip["stack_name"], + external_network, + port_name, + network.router.stack_if_name, + sec_group_id) + self.floating_ip_assoc["stack_name"] = \ + server_name + "-fip-assoc" + template.add_floating_ip_association( + self.floating_ip_assoc["stack_name"], + self.floating_ip["stack_name"], + port_name) + if self.flavor: + if isinstance(self.flavor, dict): + self.flavor["name"] = \ + self.flavor.setdefault("name", self.stack_name + "-flavor") + template.add_flavor(**self.flavor) + self.flavor_name = self.flavor["name"] + else: + self.flavor_name = self.flavor + + if self.volume: + if isinstance(self.volume, dict): + self.volume["name"] = \ + self.volume.setdefault("name", server_name + "-volume") + template.add_volume(**self.volume) + template.add_volume_attachment(server_name, self.volume["name"], + mountpoint=self.volume_mountpoint) + else: + template.add_volume_attachment(server_name, self.volume, + mountpoint=self.volume_mountpoint) + + template.add_server(server_name, self.image, flavor=self.flavor_name, + flavors=self._context.flavors, ports=port_name_list, + networks=self.networks, + scheduler_hints=scheduler_hints, user=self.user, + key_name=self.keypair_name, user_data=self.user_data, + availability_zone=self.availability_zone) + + def add_to_template(self, template, networks, scheduler_hints=None): + """adds to the template one or more servers (instances)""" + if self.instances == 1: + server_name = self.stack_name + self._add_instance(template, server_name, networks, + scheduler_hints=scheduler_hints) + else: + # TODO(hafe) fix or remove, no test/sample for this + for i in range(self.instances): + server_name = "%s-%d" % (self.stack_name, i) + self._add_instance(template, server_name, networks, + scheduler_hints=scheduler_hints) + + +def update_scheduler_hints(scheduler_hints, added_servers, placement_group): + """update scheduler hints from server's placement configuration + TODO: this code is openstack specific and should move somewhere else + """ + if placement_group.policy == "affinity": + if "same_host" in scheduler_hints: + host_list = scheduler_hints["same_host"] + else: + host_list = scheduler_hints["same_host"] = [] + else: + if "different_host" in scheduler_hints: + host_list = scheduler_hints["different_host"] + else: + host_list = scheduler_hints["different_host"] = [] + + for name in added_servers: + if name in placement_group.members: + host_list.append({'get_resource': name}) diff --git a/vnftest/core/task.py b/vnftest/core/task.py index a9718bb..8797eee 100644 --- a/vnftest/core/task.py +++ b/vnftest/core/task.py @@ -25,6 +25,7 @@ import copy import logging import sys import time +import traceback import uuid import ipaddress @@ -32,6 +33,7 @@ import os import yaml from jinja2 import Environment from six.moves import filter + from vnftest.runners import base as base_runner from vnftest.contexts.base import Context @@ -160,6 +162,7 @@ class Task(object): # pragma: no cover one_task_end_time - one_task_start_time) except Exception as e: LOG.error("Task fatal error: %s", e) + traceback.print_exc() self.task_info.task_fatal() finally: self.task_info.task_end() @@ -170,11 +173,6 @@ class Task(object): # pragma: no cover total_end_time = time.time() LOG.info("Total finished in %d secs", total_end_time - total_start_time) - - step = steps[0] - LOG.info("To generate report, execute command " - "'vnftest report generate %(task_id)s %(tc)s'", step) - LOG.info("Task ALL DONE, exiting") return self.task_info.result() def _generate_reporting(self): @@ -279,6 +277,7 @@ class Task(object): # pragma: no cover return result except Exception as e: LOG.exception('Case fatal error: %s', e) + traceback.print_exc() self.task_info.testcase_fatal(case_name) finally: self.task_info.testcase_end(case_name) @@ -316,6 +315,7 @@ class Task(object): # pragma: no cover LOG.info("Starting runner of type '%s'", runner_cfg["type"]) # Previous steps output is the input of the next step. inputs.update(self.outputs) + _resolve_step_options(step_cfg, self.contexts, inputs) runner.run(step_cfg, self.contexts, inputs) return runner @@ -330,6 +330,22 @@ class Task(object): # pragma: no cover result.extend(step_result_list) +def _resolve_step_options(step_cfg, contexts, inputs): + inputs = copy.deepcopy(inputs) + contexts_dict = {} + inputs['context'] = contexts_dict + if contexts is not None: + for context in contexts: + context_as_dict = utils.normalize_data_struct(context) + contexts_dict[context.assigned_name] = context_as_dict + options = step_cfg.get("options", {}) + resolved_options = {} + for k, v in options.items(): + v = utils.format(v, inputs) + resolved_options[k] = v + step_cfg["options"] = resolved_options + + class TaskParser(object): # pragma: no cover """Parser for task config files in yaml format""" diff --git a/vnftest/orchestrator/__init__.py b/vnftest/orchestrator/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/vnftest/orchestrator/__init__.py diff --git a/vnftest/orchestrator/heat.py b/vnftest/orchestrator/heat.py new file mode 100644 index 0000000..14c6d73 --- /dev/null +++ b/vnftest/orchestrator/heat.py @@ -0,0 +1,661 @@ +############################################################################## +# Copyright 2018 EuropeanSoftwareMarketingLtd. +# =================================================================== +# Licensed under the ApacheLicense, Version2.0 (the"License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and limitations under +# the License +############################################################################## +# vnftest comment: this is a modified copy of +# yardstick/orchestrator/heat.py +"""Heat template and stack management""" + +from __future__ import absolute_import +import collections +import datetime +import getpass +import logging +import pkg_resources +import pprint +import socket +import tempfile +import time + +from oslo_serialization import jsonutils +from oslo_utils import encodeutils +from shade._heat import event_utils + +from vnftest.common import constants as consts +from vnftest.common import exceptions +from vnftest.common import template_format +from vnftest.common import openstack_utils as op_utils + + +log = logging.getLogger(__name__) + + +PROVIDER_SRIOV = "sriov" + +_DEPLOYED_STACKS = {} + + +class HeatStack(object): + """Represents a Heat stack (deployed template) """ + + def __init__(self, name, os_cloud_config=None): + self.name = name + self.outputs = {} + os_cloud_config = {} if not os_cloud_config else os_cloud_config + self._cloud = op_utils.get_shade_client(**os_cloud_config) + self._stack = None + + def _update_stack_tracking(self): + outputs = self._stack.outputs + self.outputs = {output['output_key']: output['output_value'] for output + in outputs} + if self.uuid: + _DEPLOYED_STACKS[self.uuid] = self._stack + + def create(self, template, heat_parameters, wait, timeout): + """Creates an OpenStack stack from a template""" + with tempfile.NamedTemporaryFile('wb', delete=False) as template_file: + template_file.write(jsonutils.dump_as_bytes(template)) + template_file.close() + self._stack = self._cloud.create_stack( + self.name, template_file=template_file.name, wait=wait, + timeout=timeout, **heat_parameters) + + self._update_stack_tracking() + + def get_failures(self): + return event_utils.get_events(self._cloud, self._stack.id, + event_args={'resource_status': 'FAILED'}) + + def get(self): + """Retrieves an existing stack from the target cloud + + Returns a bool indicating whether the stack exists in the target cloud + If the stack exists, it will be stored as self._stack + """ + self._stack = self._cloud.get_stack(self.name) + if not self._stack: + return False + + self._update_stack_tracking() + return True + + @staticmethod + def stacks_exist(): + """Check if any stack has been deployed""" + return len(_DEPLOYED_STACKS) > 0 + + def delete(self, wait=True): + """Deletes a stack in the target cloud""" + if self.uuid is None: + return + + try: + ret = self._cloud.delete_stack(self.uuid, wait=wait) + except TypeError: + # NOTE(ralonsoh): this exception catch solves a bug in Shade, which + # tries to retrieve and read the stack status when it's already + # deleted. + ret = True + + _DEPLOYED_STACKS.pop(self.uuid) + self._stack = None + return ret + + @staticmethod + def delete_all(): + """Delete all deployed stacks""" + for stack in _DEPLOYED_STACKS: + stack.delete() + + @property + def status(self): + """Retrieve the current stack status""" + if self._stack: + return self._stack.status + + @property + def uuid(self): + """Retrieve the current stack ID""" + if self._stack: + return self._stack.id + + +class HeatTemplate(object): + """Describes a Heat template and a method to deploy template to a stack""" + + DESCRIPTION_TEMPLATE = """ +Stack built by the vnftest framework for %s on host %s %s. +All referred generated resources are prefixed with the template +name (i.e. %s). +""" + + HEAT_WAIT_LOOP_INTERVAL = 2 + HEAT_STATUS_COMPLETE = 'COMPLETE' + + def _init_template(self): + timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + self._template = { + 'heat_template_version': '2013-05-23', + 'description': self.DESCRIPTION_TEMPLATE % ( + getpass.getuser(), + socket.gethostname(), + timestamp, + self.name + ), + 'resources': {}, + 'outputs': {} + } + + # short hand for resources part of template + self.resources = self._template['resources'] + + def __init__(self, name, template_file=None, heat_parameters=None, + os_cloud_config=None): + self.name = name + self.keystone_client = None + self.heat_parameters = {} + self._os_cloud_config = {} if not os_cloud_config else os_cloud_config + + # heat_parameters is passed to heat in stack create, empty dict when + # vnftest creates the template (no get_param in resources part) + if heat_parameters: + self.heat_parameters = heat_parameters + + if template_file: + with open(template_file) as stream: + log.info('Parsing external template: %s', template_file) + template_str = stream.read() + self._template = template_format.parse(template_str) + self._parameters = heat_parameters + else: + self._init_template() + + log.debug("template object '%s' created", name) + + def add_flavor(self, name, vcpus=1, ram=1024, disk=1, ephemeral=0, + is_public=True, rxtx_factor=1.0, swap=0, + extra_specs=None): + """add to the template a Flavor description""" + if name is None: + name = 'auto' + log.debug("adding Nova::Flavor '%s' vcpus '%d' ram '%d' disk '%d' " + "ephemeral '%d' is_public '%s' rxtx_factor '%d' " + "swap '%d' extra_specs '%s'", + name, vcpus, ram, disk, ephemeral, is_public, + rxtx_factor, swap, str(extra_specs)) + + if extra_specs: + assert isinstance(extra_specs, collections.Mapping) + + self.resources[name] = { + 'type': 'OS::Nova::Flavor', + 'properties': {'name': name, + 'disk': disk, + 'vcpus': vcpus, + 'swap': swap, + 'flavorid': name, + 'rxtx_factor': rxtx_factor, + 'ram': ram, + 'is_public': is_public, + 'ephemeral': ephemeral, + 'extra_specs': extra_specs} + } + + self._template['outputs'][name] = { + 'description': 'Flavor %s ID' % name, + 'value': {'get_resource': name} + } + + def add_volume(self, name, size=10): + """add to the template a volume description""" + log.debug("adding Cinder::Volume '%s' size '%d' ", name, size) + + self.resources[name] = { + 'type': 'OS::Cinder::Volume', + 'properties': {'name': name, + 'size': size} + } + + self._template['outputs'][name] = { + 'description': 'Volume %s ID' % name, + 'value': {'get_resource': name} + } + + def add_volume_attachment(self, server_name, volume_name, mountpoint=None): + """add to the template an association of volume to instance""" + log.debug("adding Cinder::VolumeAttachment server '%s' volume '%s' ", + server_name, volume_name) + name = "%s-%s" % (server_name, volume_name) + volume_id = {'get_resource': volume_name} + self.resources[name] = { + 'type': 'OS::Cinder::VolumeAttachment', + 'properties': {'instance_uuid': {'get_resource': server_name}, + 'volume_id': volume_id} + } + + if mountpoint: + self.resources[name]['properties']['mountpoint'] = mountpoint + + def add_network(self, name, physical_network='physnet1', provider=None, + segmentation_id=None, port_security_enabled=None, network_type=None): + """add to the template a Neutron Net""" + log.debug("adding Neutron::Net '%s'", name) + if provider is None: + self.resources[name] = { + 'type': 'OS::Neutron::Net', + 'properties': { + 'name': name, + } + } + else: + self.resources[name] = { + 'type': 'OS::Neutron::ProviderNet', + 'properties': { + 'name': name, + 'network_type': 'flat' if network_type is None else network_type, + 'physical_network': physical_network, + }, + } + if segmentation_id: + self.resources[name]['properties']['segmentation_id'] = segmentation_id + if network_type is None: + self.resources[name]['properties']['network_type'] = 'vlan' + # if port security is not defined then don't add to template: + # some deployments don't have port security plugin installed + if port_security_enabled is not None: + self.resources[name]['properties']['port_security_enabled'] = port_security_enabled + + def add_server_group(self, name, policies): # pragma: no cover + """add to the template a ServerGroup""" + log.debug("adding Nova::ServerGroup '%s'", name) + policies = policies if isinstance(policies, list) else [policies] + self.resources[name] = { + 'type': 'OS::Nova::ServerGroup', + 'properties': {'name': name, + 'policies': policies} + } + + def add_subnet(self, name, network, cidr, enable_dhcp='true', gateway_ip=None): + """add to the template a Neutron Subnet + """ + log.debug("adding Neutron::Subnet '%s' in network '%s', cidr '%s'", + name, network, cidr) + self.resources[name] = { + 'type': 'OS::Neutron::Subnet', + 'depends_on': network, + 'properties': { + 'name': name, + 'cidr': cidr, + 'network_id': {'get_resource': network}, + 'enable_dhcp': enable_dhcp, + } + } + if gateway_ip == 'null': + self.resources[name]['properties']['gateway_ip'] = None + elif gateway_ip is not None: + self.resources[name]['properties']['gateway_ip'] = gateway_ip + + self._template['outputs'][name] = { + 'description': 'subnet %s ID' % name, + 'value': {'get_resource': name} + } + self._template['outputs'][name + "-cidr"] = { + 'description': 'subnet %s cidr' % name, + 'value': {'get_attr': [name, 'cidr']} + } + self._template['outputs'][name + "-gateway_ip"] = { + 'description': 'subnet %s gateway_ip' % name, + 'value': {'get_attr': [name, 'gateway_ip']} + } + + def add_router(self, name, ext_gw_net, subnet_name): + """add to the template a Neutron Router and interface""" + log.debug("adding Neutron::Router:'%s', gw-net:'%s'", name, ext_gw_net) + self.resources[name] = { + 'type': 'OS::Neutron::Router', + 'depends_on': [subnet_name], + 'properties': { + 'name': name, + 'external_gateway_info': { + 'network': ext_gw_net + } + } + } + + def add_router_interface(self, name, router_name, subnet_name): + """add to the template a Neutron RouterInterface and interface""" + log.debug("adding Neutron::RouterInterface '%s' router:'%s', " + "subnet:'%s'", name, router_name, subnet_name) + self.resources[name] = { + 'type': 'OS::Neutron::RouterInterface', + 'depends_on': [router_name, subnet_name], + 'properties': { + 'router_id': {'get_resource': router_name}, + 'subnet_id': {'get_resource': subnet_name} + } + } + + def add_port(self, name, network, sec_group_id=None, + provider=None, allowed_address_pairs=None): + """add to the template a named Neutron Port + """ + net_is_existing = network.net_flags.get(consts.IS_EXISTING) + depends_on = [] if net_is_existing else [network.subnet_stack_name] + fixed_ips = [{'subnet': network.subnet}] if net_is_existing else [ + {'subnet': {'get_resource': network.subnet_stack_name}}] + network_ = network.name if net_is_existing else { + 'get_resource': network.stack_name} + self.resources[name] = { + 'type': 'OS::Neutron::Port', + 'depends_on': depends_on, + 'properties': { + 'name': name, + 'binding:vnic_type': network.vnic_type, + 'fixed_ips': fixed_ips, + 'network': network_, + } + } + + if provider == PROVIDER_SRIOV: + self.resources[name]['properties']['binding:vnic_type'] = \ + 'direct' + + if sec_group_id: + self.resources[name]['depends_on'].append(sec_group_id) + self.resources[name]['properties']['security_groups'] = \ + [sec_group_id] + + if allowed_address_pairs: + self.resources[name]['properties'][ + 'allowed_address_pairs'] = allowed_address_pairs + + log.debug("adding Neutron::Port %s", self.resources[name]) + + self._template['outputs'][name] = { + 'description': 'Address for interface %s' % name, + 'value': {'get_attr': [name, 'fixed_ips', 0, 'ip_address']} + } + self._template['outputs'][name + "-subnet_id"] = { + 'description': 'Address for interface %s' % name, + 'value': {'get_attr': [name, 'fixed_ips', 0, 'subnet_id']} + } + self._template['outputs'][name + "-mac_address"] = { + 'description': 'MAC Address for interface %s' % name, + 'value': {'get_attr': [name, 'mac_address']} + } + self._template['outputs'][name + "-device_id"] = { + 'description': 'Device ID for interface %s' % name, + 'value': {'get_attr': [name, 'device_id']} + } + self._template['outputs'][name + "-network_id"] = { + 'description': 'Network ID for interface %s' % name, + 'value': {'get_attr': [name, 'network_id']} + } + + def add_floating_ip(self, name, network_name, port_name, router_if_name, + secgroup_name=None): + """add to the template a Nova FloatingIP resource + see: https://bugs.launchpad.net/heat/+bug/1299259 + """ + log.debug("adding Nova::FloatingIP '%s', network '%s', port '%s', " + "rif '%s'", name, network_name, port_name, router_if_name) + + self.resources[name] = { + 'type': 'OS::Nova::FloatingIP', + 'depends_on': [port_name, router_if_name], + 'properties': { + 'pool': network_name + } + } + + if secgroup_name: + self.resources[name]["depends_on"].append(secgroup_name) + + self._template['outputs'][name] = { + 'description': 'floating ip %s' % name, + 'value': {'get_attr': [name, 'ip']} + } + + def add_floating_ip_association(self, name, floating_ip_name, port_name): + """add to the template a Nova FloatingIP Association resource + """ + log.debug("adding Nova::FloatingIPAssociation '%s', server '%s', " + "floating_ip '%s'", name, port_name, floating_ip_name) + + self.resources[name] = { + 'type': 'OS::Neutron::FloatingIPAssociation', + 'depends_on': [port_name], + 'properties': { + 'floatingip_id': {'get_resource': floating_ip_name}, + 'port_id': {'get_resource': port_name} + } + } + + def add_keypair(self, name, key_id): + """add to the template a Nova KeyPair""" + log.debug("adding Nova::KeyPair '%s'", name) + self.resources[name] = { + 'type': 'OS::Nova::KeyPair', + 'properties': { + 'name': name, + # resource_string returns bytes, so we must decode to unicode + 'public_key': encodeutils.safe_decode( + pkg_resources.resource_string( + 'vnftest.resources', + 'files/vnftest_key-' + + key_id + '.pub'), + 'utf-8') + } + } + + def add_servergroup(self, name, policy): + """add to the template a Nova ServerGroup""" + log.debug("adding Nova::ServerGroup '%s', policy '%s'", name, policy) + if policy not in ["anti-affinity", "affinity"]: + raise ValueError(policy) + + self.resources[name] = { + 'type': 'OS::Nova::ServerGroup', + 'properties': { + 'name': name, + 'policies': [policy] + } + } + + self._template['outputs'][name] = { + 'description': 'ID Server Group %s' % name, + 'value': {'get_resource': name} + } + + def add_security_group(self, name, security_group=None): + """add to the template a Neutron SecurityGroup""" + log.debug("adding Neutron::SecurityGroup '%s'", name) + description = ("Group allowing IPv4 and IPv6 for icmp and upd/tcp on" + "all ports") + rules = [ + {'remote_ip_prefix': '0.0.0.0/0', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'protocol': 'icmp'}, + {'remote_ip_prefix': '::/0', + 'ethertype': 'IPv6', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'ethertype': 'IPv6', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'ethertype': 'IPv6', + 'protocol': 'ipv6-icmp'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'direction': 'egress', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'direction': 'egress', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'direction': 'egress', + 'protocol': 'icmp'}, + {'remote_ip_prefix': '::/0', + 'direction': 'egress', + 'ethertype': 'IPv6', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'direction': 'egress', + 'ethertype': 'IPv6', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'direction': 'egress', + 'ethertype': 'IPv6', + 'protocol': 'ipv6-icmp'}, + ] + if security_group: + description = "Custom security group rules defined by the user" + rules = security_group.get('rules') + + log.debug("The security group rules is %s", rules) + + self.resources[name] = { + 'type': 'OS::Neutron::SecurityGroup', + 'properties': { + 'name': name, + 'description': description, + 'rules': rules + } + } + + self._template['outputs'][name] = { + 'description': 'ID of Security Group', + 'value': {'get_resource': name} + } + + def add_server(self, name, image, flavor, flavors, ports=None, networks=None, + scheduler_hints=None, user=None, key_name=None, user_data=None, metadata=None, + additional_properties=None, availability_zone=None): + """add to the template a Nova Server """ + log.debug("adding Nova::Server '%s', image '%s', flavor '%s', " + "ports %s", name, image, flavor, ports) + + self.resources[name] = { + 'type': 'OS::Nova::Server', + 'depends_on': [] + } + + server_properties = { + 'name': name, + 'image': image, + 'flavor': {}, + 'networks': [] # list of dictionaries + } + if availability_zone: + server_properties["availability_zone"] = availability_zone + + if flavor in flavors: + self.resources[name]['depends_on'].append(flavor) + server_properties["flavor"] = {'get_resource': flavor} + else: + server_properties["flavor"] = flavor + + if user: + server_properties['admin_user'] = user + + if key_name: + self.resources[name]['depends_on'].append(key_name) + server_properties['key_name'] = {'get_resource': key_name} + + if ports: + self.resources[name]['depends_on'].extend(ports) + for port in ports: + server_properties['networks'].append( + {'port': {'get_resource': port}} + ) + + if networks: + for n in networks: + server_properties['networks'].append(n) + if 'network' in n: + network_name = n['network'] + self._template['outputs'][name + ".networks.%s-ip" % network_name] = { + 'description': 'network %s ip' % network_name, + 'value': {'get_attr': [name, 'networks', network_name, 0]} + } + + if scheduler_hints: + server_properties['scheduler_hints'] = scheduler_hints + + if user_data: + server_properties['user_data'] = user_data + + if metadata: + assert isinstance(metadata, collections.Mapping) + server_properties['metadata'] = metadata + + if additional_properties: + assert isinstance(additional_properties, collections.Mapping) + for prop in additional_properties: + server_properties[prop] = additional_properties[prop] + + server_properties['config_drive'] = True + + self.resources[name]['properties'] = server_properties + + self._template['outputs'][name] = { + 'description': 'VM UUID', + 'value': {'get_resource': name} + } + + def create(self, block=True, timeout=3600): + """Creates a stack in the target based on the stored template + + :param block: (bool) Wait for Heat create to finish + :param timeout: (int) Timeout in seconds for Heat create, + default 3600s + :return A dict with the requested output values from the template + """ + log.info("Creating stack '%s' START", self.name) + + start_time = time.time() + stack = HeatStack(self.name, os_cloud_config=self._os_cloud_config) + stack.create(self._template, self.heat_parameters, block, timeout) + + if not block: + log.info("Creating stack '%s' DONE in %d secs", + self.name, time.time() - start_time) + return stack + + if stack.status != self.HEAT_STATUS_COMPLETE: + for event in stack.get_failures(): + log.error("%s", event.resource_status_reason) + log.error(pprint.pformat(self._template)) + raise exceptions.HeatTemplateError(stack_name=self.name) + + log.info("Creating stack '%s' DONE in %d secs", + self.name, time.time() - start_time) + return stack diff --git a/vnftest/tests/unit/common/test_utils.py b/vnftest/tests/unit/common/test_utils.py index d152a98..6f82537 100644 --- a/vnftest/tests/unit/common/test_utils.py +++ b/vnftest/tests/unit/common/test_utils.py @@ -798,19 +798,6 @@ power management: assert sockets == [0, 1] -class ChangeObjToDictTestCase(unittest.TestCase): - - def test_change_obj_to_dict(self): - class A(object): - def __init__(self): - self.name = 'vnftest' - - obj = A() - obj_r = utils.change_obj_to_dict(obj) - obj_s = {'name': 'vnftest'} - self.assertEqual(obj_r, obj_s) - - class SetDictValueTestCase(unittest.TestCase): def test_set_dict_value(self): |