diff options
Diffstat (limited to 'aria/multivim-plugin/neutron_plugin')
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/__init__.py | 1 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/floatingip.py | 104 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/network.py | 109 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/port.py | 222 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/router.py | 215 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/security_group.py | 130 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/subnet.py | 101 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/tests/__init__.py | 1 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/tests/test.py | 220 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/tests/test_port.py | 156 | ||||
| -rw-r--r-- | aria/multivim-plugin/neutron_plugin/tests/test_security_group.py | 115 |
11 files changed, 1374 insertions, 0 deletions
diff --git a/aria/multivim-plugin/neutron_plugin/__init__.py b/aria/multivim-plugin/neutron_plugin/__init__.py new file mode 100644 index 0000000000..04cb21f745 --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/__init__.py @@ -0,0 +1 @@ +__author__ = 'idanmo' diff --git a/aria/multivim-plugin/neutron_plugin/floatingip.py b/aria/multivim-plugin/neutron_plugin/floatingip.py new file mode 100644 index 0000000000..1a9d0449ca --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/floatingip.py @@ -0,0 +1,104 @@ +######### +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +from cloudify import ctx +from cloudify.decorators import operation +from cloudify.exceptions import NonRecoverableError +from openstack_plugin_common import ( + with_neutron_client, + provider, + is_external_relationship, + is_external_relationship_not_conditionally_created, + OPENSTACK_ID_PROPERTY +) +from openstack_plugin_common.floatingip import ( + use_external_floatingip, + set_floatingip_runtime_properties, + delete_floatingip, + floatingip_creation_validation +) + + +@operation +@with_neutron_client +def create(neutron_client, args, **kwargs): + + if use_external_floatingip(neutron_client, 'floating_ip_address', + lambda ext_fip: ext_fip['floating_ip_address']): + return + + floatingip = { + # No defaults + } + floatingip.update(ctx.node.properties['floatingip'], **args) + + # Sugar: floating_network_name -> (resolve) -> floating_network_id + if 'floating_network_name' in floatingip: + floatingip['floating_network_id'] = neutron_client.cosmo_get_named( + 'network', floatingip['floating_network_name'])['id'] + del floatingip['floating_network_name'] + elif 'floating_network_id' not in floatingip: + provider_context = provider(ctx) + ext_network = provider_context.ext_network + if ext_network: + floatingip['floating_network_id'] = ext_network['id'] + else: + raise NonRecoverableError( + 'Missing floating network id, name or external network') + + fip = neutron_client.create_floatingip( + {'floatingip': floatingip})['floatingip'] + set_floatingip_runtime_properties(fip['id'], fip['floating_ip_address']) + + ctx.logger.info('Floating IP creation response: {0}'.format(fip)) + + +@operation +@with_neutron_client +def delete(neutron_client, **kwargs): + delete_floatingip(neutron_client) + + +@operation +@with_neutron_client +def creation_validation(neutron_client, **kwargs): + floatingip_creation_validation(neutron_client, 'floating_ip_address') + + +@operation +@with_neutron_client +def connect_port(neutron_client, **kwargs): + if is_external_relationship_not_conditionally_created(ctx): + return + + port_id = ctx.source.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + floating_ip_id = ctx.target.instance.runtime_properties[ + OPENSTACK_ID_PROPERTY] + fip = {'port_id': port_id} + neutron_client.update_floatingip(floating_ip_id, {'floatingip': fip}) + + +@operation +@with_neutron_client +def disconnect_port(neutron_client, **kwargs): + if is_external_relationship(ctx): + ctx.logger.info('Not disassociating floatingip and port since ' + 'external floatingip and port are being used') + return + + floating_ip_id = ctx.target.instance.runtime_properties[ + OPENSTACK_ID_PROPERTY] + fip = {'port_id': None} + neutron_client.update_floatingip(floating_ip_id, {'floatingip': fip}) diff --git a/aria/multivim-plugin/neutron_plugin/network.py b/aria/multivim-plugin/neutron_plugin/network.py new file mode 100644 index 0000000000..eadcc3b4e8 --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/network.py @@ -0,0 +1,109 @@ +######### +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +from cloudify import ctx +from cloudify.decorators import operation +from cloudify.exceptions import NonRecoverableError +from openstack_plugin_common import ( + transform_resource_name, + with_neutron_client, + get_resource_id, + is_external_resource, + is_external_resource_not_conditionally_created, + delete_resource_and_runtime_properties, + use_external_resource, + validate_resource, + OPENSTACK_ID_PROPERTY, + OPENSTACK_TYPE_PROPERTY, + OPENSTACK_NAME_PROPERTY, + COMMON_RUNTIME_PROPERTIES_KEYS +) + +NETWORK_OPENSTACK_TYPE = 'network' + +# Runtime properties +RUNTIME_PROPERTIES_KEYS = COMMON_RUNTIME_PROPERTIES_KEYS + + +@operation +@with_neutron_client +def create(neutron_client, args, **kwargs): + + if use_external_resource(ctx, neutron_client, NETWORK_OPENSTACK_TYPE): + return + + network = { + 'admin_state_up': True, + 'name': get_resource_id(ctx, NETWORK_OPENSTACK_TYPE), + } + network.update(ctx.node.properties['network'], **args) + transform_resource_name(ctx, network) + + net = neutron_client.create_network({'network': network})['network'] + ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY] = net['id'] + ctx.instance.runtime_properties[OPENSTACK_TYPE_PROPERTY] =\ + NETWORK_OPENSTACK_TYPE + ctx.instance.runtime_properties[OPENSTACK_NAME_PROPERTY] = net['name'] + + +@operation +@with_neutron_client +def start(neutron_client, **kwargs): + network_id = ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + + if is_external_resource_not_conditionally_created(ctx): + ctx.logger.info('Validating external network is started') + if not neutron_client.show_network( + network_id)['network']['admin_state_up']: + raise NonRecoverableError( + 'Expected external resource network {0} to be in ' + '"admin_state_up"=True'.format(network_id)) + return + + neutron_client.update_network( + network_id, { + 'network': { + 'admin_state_up': True + } + }) + + +@operation +@with_neutron_client +def stop(neutron_client, **kwargs): + if is_external_resource(ctx): + ctx.logger.info('Not stopping network since an external network is ' + 'being used') + return + + neutron_client.update_network( + ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY], { + 'network': { + 'admin_state_up': False + } + }) + + +@operation +@with_neutron_client +def delete(neutron_client, **kwargs): + delete_resource_and_runtime_properties(ctx, neutron_client, + RUNTIME_PROPERTIES_KEYS) + + +@operation +@with_neutron_client +def creation_validation(neutron_client, **kwargs): + validate_resource(ctx, neutron_client, NETWORK_OPENSTACK_TYPE) diff --git a/aria/multivim-plugin/neutron_plugin/port.py b/aria/multivim-plugin/neutron_plugin/port.py new file mode 100644 index 0000000000..4db4c442c5 --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/port.py @@ -0,0 +1,222 @@ +######### +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +from cloudify import ctx +from cloudify.decorators import operation +from cloudify.exceptions import NonRecoverableError + +import neutronclient.common.exceptions as neutron_exceptions + +from openstack_plugin_common import ( + transform_resource_name, + with_neutron_client, + with_nova_client, + get_resource_id, + get_openstack_id_of_single_connected_node_by_openstack_type, + delete_resource_and_runtime_properties, + delete_runtime_properties, + use_external_resource, + is_external_relationship, + validate_resource, + OPENSTACK_ID_PROPERTY, + OPENSTACK_TYPE_PROPERTY, + OPENSTACK_NAME_PROPERTY, + COMMON_RUNTIME_PROPERTIES_KEYS, + is_external_relationship_not_conditionally_created) + +from neutron_plugin.network import NETWORK_OPENSTACK_TYPE +from neutron_plugin.subnet import SUBNET_OPENSTACK_TYPE +from openstack_plugin_common.floatingip import get_server_floating_ip + +PORT_OPENSTACK_TYPE = 'port' + +# Runtime properties +FIXED_IP_ADDRESS_PROPERTY = 'fixed_ip_address' # the fixed ip address +MAC_ADDRESS_PROPERTY = 'mac_address' # the mac address +RUNTIME_PROPERTIES_KEYS = \ + COMMON_RUNTIME_PROPERTIES_KEYS + [FIXED_IP_ADDRESS_PROPERTY, + MAC_ADDRESS_PROPERTY] + +NO_SG_PORT_CONNECTION_RETRY_INTERVAL = 3 + + +@operation +@with_neutron_client +def create(neutron_client, args, **kwargs): + + ext_port = use_external_resource(ctx, neutron_client, PORT_OPENSTACK_TYPE) + if ext_port: + try: + net_id = \ + get_openstack_id_of_single_connected_node_by_openstack_type( + ctx, NETWORK_OPENSTACK_TYPE, True) + + if net_id: + port_id = ctx.instance.runtime_properties[ + OPENSTACK_ID_PROPERTY] + + if neutron_client.show_port( + port_id)['port']['network_id'] != net_id: + raise NonRecoverableError( + 'Expected external resources port {0} and network {1} ' + 'to be connected'.format(port_id, net_id)) + + ctx.instance.runtime_properties[FIXED_IP_ADDRESS_PROPERTY] = \ + _get_fixed_ip(ext_port) + ctx.instance.runtime_properties[MAC_ADDRESS_PROPERTY] = \ + ext_port['mac_address'] + return + except Exception: + delete_runtime_properties(ctx, RUNTIME_PROPERTIES_KEYS) + raise + + net_id = get_openstack_id_of_single_connected_node_by_openstack_type( + ctx, NETWORK_OPENSTACK_TYPE) + + port = { + 'name': get_resource_id(ctx, PORT_OPENSTACK_TYPE), + 'network_id': net_id, + 'security_groups': [], + } + + _handle_fixed_ips(port) + port.update(ctx.node.properties['port'], **args) + transform_resource_name(ctx, port) + + p = neutron_client.create_port({'port': port})['port'] + ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY] = p['id'] + ctx.instance.runtime_properties[OPENSTACK_TYPE_PROPERTY] =\ + PORT_OPENSTACK_TYPE + ctx.instance.runtime_properties[OPENSTACK_NAME_PROPERTY] = p['name'] + ctx.instance.runtime_properties[FIXED_IP_ADDRESS_PROPERTY] = \ + _get_fixed_ip(p) + ctx.instance.runtime_properties[MAC_ADDRESS_PROPERTY] = p['mac_address'] + + +@operation +@with_neutron_client +def delete(neutron_client, **kwargs): + try: + delete_resource_and_runtime_properties(ctx, neutron_client, + RUNTIME_PROPERTIES_KEYS) + except neutron_exceptions.NeutronClientException, e: + if e.status_code == 404: + # port was probably deleted when an attached device was deleted + delete_runtime_properties(ctx, RUNTIME_PROPERTIES_KEYS) + else: + raise + + +@operation +@with_nova_client +@with_neutron_client +def detach(nova_client, neutron_client, **kwargs): + + if is_external_relationship(ctx): + ctx.logger.info('Not detaching port from server since ' + 'external port and server are being used') + return + + port_id = ctx.target.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + server_id = ctx.source.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + + server_floating_ip = get_server_floating_ip(neutron_client, server_id) + if server_floating_ip: + ctx.logger.info('We have floating ip {0} attached to server' + .format(server_floating_ip['floating_ip_address'])) + server = nova_client.servers.get(server_id) + server.remove_floating_ip(server_floating_ip['floating_ip_address']) + return ctx.operation.retry( + message='Waiting for the floating ip {0} to ' + 'detach from server {1}..' + .format(server_floating_ip['floating_ip_address'], + server_id), + retry_after=10) + change = { + 'port': { + 'device_id': '', + 'device_owner': '' + } + } + ctx.logger.info('Detaching port {0}...'.format(port_id)) + neutron_client.update_port(port_id, change) + ctx.logger.info('Successfully detached port {0}'.format(port_id)) + + +@operation +@with_neutron_client +def connect_security_group(neutron_client, **kwargs): + port_id = ctx.source.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + security_group_id = ctx.target.instance.runtime_properties[ + OPENSTACK_ID_PROPERTY] + + if is_external_relationship_not_conditionally_created(ctx): + ctx.logger.info('Validating external port and security-group are ' + 'connected') + if any(sg for sg in neutron_client.show_port(port_id)['port'].get( + 'security_groups', []) if sg == security_group_id): + return + raise NonRecoverableError( + 'Expected external resources port {0} and security-group {1} to ' + 'be connected'.format(port_id, security_group_id)) + + # WARNING: non-atomic operation + port = neutron_client.cosmo_get('port', id=port_id) + ctx.logger.info( + "connect_security_group(): source_id={0} target={1}".format( + port_id, ctx.target.instance.runtime_properties)) + sgs = port['security_groups'] + [security_group_id] + neutron_client.update_port(port_id, {'port': {'security_groups': sgs}}) + + # Double check if SG has been actually updated (a race-condition + # in OpenStack): + port_info = neutron_client.show_port(port_id)['port'] + port_security_groups = port_info.get('security_groups', []) + if security_group_id not in port_security_groups: + return ctx.operation.retry( + message='Security group connection (`{0}\' -> `{1}\')' + ' has not been established!'.format(port_id, + security_group_id), + retry_after=NO_SG_PORT_CONNECTION_RETRY_INTERVAL + ) + + +@operation +@with_neutron_client +def creation_validation(neutron_client, **kwargs): + validate_resource(ctx, neutron_client, PORT_OPENSTACK_TYPE) + + +def _get_fixed_ip(port): + # a port may have no fixed IP if it's set on a network without subnets + return port['fixed_ips'][0]['ip_address'] if port['fixed_ips'] else None + + +def _handle_fixed_ips(port): + fixed_ips_element = {} + + # checking for fixed ip property + if ctx.node.properties['fixed_ip']: + fixed_ips_element['ip_address'] = ctx.node.properties['fixed_ip'] + + # checking for a connected subnet + subnet_id = get_openstack_id_of_single_connected_node_by_openstack_type( + ctx, SUBNET_OPENSTACK_TYPE, if_exists=True) + if subnet_id: + fixed_ips_element['subnet_id'] = subnet_id + + # applying fixed ip parameter, if available + if fixed_ips_element: + port['fixed_ips'] = [fixed_ips_element] diff --git a/aria/multivim-plugin/neutron_plugin/router.py b/aria/multivim-plugin/neutron_plugin/router.py new file mode 100644 index 0000000000..1a2851e4bc --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/router.py @@ -0,0 +1,215 @@ +######### +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +import warnings + +from cloudify import ctx +from cloudify.decorators import operation +from cloudify.exceptions import NonRecoverableError + +from openstack_plugin_common import ( + provider, + transform_resource_name, + get_resource_id, + with_neutron_client, + use_external_resource, + is_external_relationship, + is_external_relationship_not_conditionally_created, + delete_runtime_properties, + get_openstack_ids_of_connected_nodes_by_openstack_type, + delete_resource_and_runtime_properties, + get_resource_by_name_or_id, + validate_resource, + COMMON_RUNTIME_PROPERTIES_KEYS, + OPENSTACK_ID_PROPERTY, + OPENSTACK_TYPE_PROPERTY, + OPENSTACK_NAME_PROPERTY +) + +from neutron_plugin.network import NETWORK_OPENSTACK_TYPE + + +ROUTER_OPENSTACK_TYPE = 'router' + +# Runtime properties +RUNTIME_PROPERTIES_KEYS = COMMON_RUNTIME_PROPERTIES_KEYS + + +@operation +@with_neutron_client +def create(neutron_client, args, **kwargs): + + if use_external_resource(ctx, neutron_client, ROUTER_OPENSTACK_TYPE): + try: + ext_net_id_by_rel = _get_connected_ext_net_id(neutron_client) + + if ext_net_id_by_rel: + router_id = \ + ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + + router = neutron_client.show_router(router_id)['router'] + if not (router['external_gateway_info'] and 'network_id' in + router['external_gateway_info'] and + router['external_gateway_info']['network_id'] == + ext_net_id_by_rel): + raise NonRecoverableError( + 'Expected external resources router {0} and ' + 'external network {1} to be connected'.format( + router_id, ext_net_id_by_rel)) + return + except Exception: + delete_runtime_properties(ctx, RUNTIME_PROPERTIES_KEYS) + raise + + router = { + 'name': get_resource_id(ctx, ROUTER_OPENSTACK_TYPE), + } + router.update(ctx.node.properties['router'], **args) + transform_resource_name(ctx, router) + + _handle_external_network_config(router, neutron_client) + + r = neutron_client.create_router({'router': router})['router'] + + ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY] = r['id'] + ctx.instance.runtime_properties[OPENSTACK_TYPE_PROPERTY] =\ + ROUTER_OPENSTACK_TYPE + ctx.instance.runtime_properties[OPENSTACK_NAME_PROPERTY] = r['name'] + + +@operation +@with_neutron_client +def connect_subnet(neutron_client, **kwargs): + router_id = ctx.target.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + subnet_id = ctx.source.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + + if is_external_relationship_not_conditionally_created(ctx): + ctx.logger.info('Validating external subnet and router ' + 'are associated') + for port in neutron_client.list_ports(device_id=router_id)['ports']: + for fixed_ip in port.get('fixed_ips', []): + if fixed_ip.get('subnet_id') == subnet_id: + return + raise NonRecoverableError( + 'Expected external resources router {0} and subnet {1} to be ' + 'connected'.format(router_id, subnet_id)) + + neutron_client.add_interface_router(router_id, {'subnet_id': subnet_id}) + + +@operation +@with_neutron_client +def disconnect_subnet(neutron_client, **kwargs): + if is_external_relationship(ctx): + ctx.logger.info('Not connecting subnet and router since external ' + 'subnet and router are being used') + return + + neutron_client.remove_interface_router( + ctx.target.instance.runtime_properties[OPENSTACK_ID_PROPERTY], { + 'subnet_id': ctx.source.instance.runtime_properties[ + OPENSTACK_ID_PROPERTY] + } + ) + + +@operation +@with_neutron_client +def delete(neutron_client, **kwargs): + delete_resource_and_runtime_properties(ctx, neutron_client, + RUNTIME_PROPERTIES_KEYS) + + +@operation +@with_neutron_client +def creation_validation(neutron_client, **kwargs): + validate_resource(ctx, neutron_client, ROUTER_OPENSTACK_TYPE) + + +def _insert_ext_net_id_to_router_config(ext_net_id, router): + router['external_gateway_info'] = router.get( + 'external_gateway_info', {}) + router['external_gateway_info']['network_id'] = ext_net_id + + +def _handle_external_network_config(router, neutron_client): + # attempting to find an external network for the router to connect to - + # first by either a network name or id passed in explicitly; then by a + # network connected by a relationship; with a final optional fallback to an + # external network set in the Provider-context. Otherwise the router will + # simply not get connected to an external network + + provider_context = provider(ctx) + + ext_net_id_by_rel = _get_connected_ext_net_id(neutron_client) + ext_net_by_property = ctx.node.properties['external_network'] + + # the following is meant for backwards compatibility with the + # 'network_name' sugaring + if 'external_gateway_info' in router and 'network_name' in \ + router['external_gateway_info']: + warnings.warn( + 'Passing external "network_name" inside the ' + 'external_gateway_info key of the "router" property is now ' + 'deprecated; Use the "external_network" property instead', + DeprecationWarning) + + ext_net_by_property = router['external_gateway_info']['network_name'] + del (router['external_gateway_info']['network_name']) + + # need to check if the user explicitly passed network_id in the external + # gateway configuration as it affects external network behavior by + # relationship and/or provider context + if 'external_gateway_info' in router and 'network_id' in \ + router['external_gateway_info']: + ext_net_by_property = router['external_gateway_info']['network_name'] + + if ext_net_by_property and ext_net_id_by_rel: + raise RuntimeError( + "Router can't have an external network connected by both a " + 'relationship and by a network name/id') + + if ext_net_by_property: + ext_net_id = get_resource_by_name_or_id( + ext_net_by_property, NETWORK_OPENSTACK_TYPE, neutron_client)['id'] + _insert_ext_net_id_to_router_config(ext_net_id, router) + elif ext_net_id_by_rel: + _insert_ext_net_id_to_router_config(ext_net_id_by_rel, router) + elif ctx.node.properties['default_to_managers_external_network'] and \ + provider_context.ext_network: + _insert_ext_net_id_to_router_config(provider_context.ext_network['id'], + router) + + +def _check_if_network_is_external(neutron_client, network_id): + return neutron_client.show_network( + network_id)['network']['router:external'] + + +def _get_connected_ext_net_id(neutron_client): + ext_net_ids = \ + [net_id + for net_id in + get_openstack_ids_of_connected_nodes_by_openstack_type( + ctx, NETWORK_OPENSTACK_TYPE) if + _check_if_network_is_external(neutron_client, net_id)] + + if len(ext_net_ids) > 1: + raise NonRecoverableError( + 'More than one external network is connected to router {0}' + ' by a relationship; External network IDs: {0}'.format( + ext_net_ids)) + + return ext_net_ids[0] if ext_net_ids else None diff --git a/aria/multivim-plugin/neutron_plugin/security_group.py b/aria/multivim-plugin/neutron_plugin/security_group.py new file mode 100644 index 0000000000..5f335f482b --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/security_group.py @@ -0,0 +1,130 @@ +######### +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +from time import sleep + +from requests.exceptions import RequestException + +from cloudify import ctx +from cloudify.decorators import operation +from cloudify.exceptions import NonRecoverableError +from openstack_plugin_common import ( + transform_resource_name, + with_neutron_client, + delete_resource_and_runtime_properties, +) +from openstack_plugin_common.security_group import ( + build_sg_data, + process_rules, + use_external_sg, + set_sg_runtime_properties, + delete_sg, + sg_creation_validation, + RUNTIME_PROPERTIES_KEYS +) + +DEFAULT_RULE_VALUES = { + 'direction': 'ingress', + 'ethertype': 'IPv4', + 'port_range_min': 1, + 'port_range_max': 65535, + 'protocol': 'tcp', + 'remote_group_id': None, + 'remote_ip_prefix': '0.0.0.0/0', +} + + +@operation +@with_neutron_client +def create( + neutron_client, args, + status_attempts=10, status_timeout=2, **kwargs +): + + security_group = build_sg_data(args) + if not security_group['description']: + security_group['description'] = ctx.node.properties['description'] + + sg_rules = process_rules(neutron_client, DEFAULT_RULE_VALUES, + 'remote_ip_prefix', 'remote_group_id', + 'port_range_min', 'port_range_max') + + disable_default_egress_rules = ctx.node.properties.get( + 'disable_default_egress_rules') + + if use_external_sg(neutron_client): + return + + transform_resource_name(ctx, security_group) + + sg = neutron_client.create_security_group( + {'security_group': security_group})['security_group'] + + for attempt in range(max(status_attempts, 1)): + sleep(status_timeout) + try: + neutron_client.show_security_group(sg['id']) + except RequestException as e: + ctx.logger.debug("Waiting for SG to be visible. Attempt {}".format( + attempt)) + else: + break + else: + raise NonRecoverableError( + "Timed out waiting for security_group to exist", e) + + set_sg_runtime_properties(sg, neutron_client) + + try: + if disable_default_egress_rules: + for er in _egress_rules(_rules_for_sg_id(neutron_client, + sg['id'])): + neutron_client.delete_security_group_rule(er['id']) + + for sgr in sg_rules: + sgr['security_group_id'] = sg['id'] + neutron_client.create_security_group_rule( + {'security_group_rule': sgr}) + except Exception: + try: + delete_resource_and_runtime_properties( + ctx, neutron_client, + RUNTIME_PROPERTIES_KEYS) + except Exception as e: + raise NonRecoverableError( + 'Exception while tearing down for retry', e) + raise + + +@operation +@with_neutron_client +def delete(neutron_client, **kwargs): + delete_sg(neutron_client) + + +@operation +@with_neutron_client +def creation_validation(neutron_client, **kwargs): + sg_creation_validation(neutron_client, 'remote_ip_prefix') + + +def _egress_rules(rules): + return [rule for rule in rules if rule.get('direction') == 'egress'] + + +def _rules_for_sg_id(neutron_client, id): + rules = neutron_client.list_security_group_rules()['security_group_rules'] + rules = [rule for rule in rules if rule['security_group_id'] == id] + return rules diff --git a/aria/multivim-plugin/neutron_plugin/subnet.py b/aria/multivim-plugin/neutron_plugin/subnet.py new file mode 100644 index 0000000000..6e97c96755 --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/subnet.py @@ -0,0 +1,101 @@ +######### +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +from cloudify import ctx +from cloudify.decorators import operation +from cloudify.exceptions import NonRecoverableError +from openstack_plugin_common import ( + with_neutron_client, + transform_resource_name, + get_resource_id, + get_openstack_id_of_single_connected_node_by_openstack_type, + delete_resource_and_runtime_properties, + delete_runtime_properties, + use_external_resource, + validate_resource, + validate_ip_or_range_syntax, + OPENSTACK_ID_PROPERTY, + OPENSTACK_TYPE_PROPERTY, + OPENSTACK_NAME_PROPERTY, + COMMON_RUNTIME_PROPERTIES_KEYS +) + +from neutron_plugin.network import NETWORK_OPENSTACK_TYPE + +SUBNET_OPENSTACK_TYPE = 'subnet' + +# Runtime properties +RUNTIME_PROPERTIES_KEYS = COMMON_RUNTIME_PROPERTIES_KEYS + + +@operation +@with_neutron_client +def create(neutron_client, args, **kwargs): + + if use_external_resource(ctx, neutron_client, SUBNET_OPENSTACK_TYPE): + try: + net_id = \ + get_openstack_id_of_single_connected_node_by_openstack_type( + ctx, NETWORK_OPENSTACK_TYPE, True) + + if net_id: + subnet_id = \ + ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY] + + if neutron_client.show_subnet( + subnet_id)['subnet']['network_id'] != net_id: + raise NonRecoverableError( + 'Expected external resources subnet {0} and network' + ' {1} to be connected'.format(subnet_id, net_id)) + return + except Exception: + delete_runtime_properties(ctx, RUNTIME_PROPERTIES_KEYS) + raise + + net_id = get_openstack_id_of_single_connected_node_by_openstack_type( + ctx, NETWORK_OPENSTACK_TYPE) + subnet = { + 'name': get_resource_id(ctx, SUBNET_OPENSTACK_TYPE), + 'network_id': net_id, + } + subnet.update(ctx.node.properties['subnet'], **args) + transform_resource_name(ctx, subnet) + + s = neutron_client.create_subnet({'subnet': subnet})['subnet'] + ctx.instance.runtime_properties[OPENSTACK_ID_PROPERTY] = s['id'] + ctx.instance.runtime_properties[OPENSTACK_TYPE_PROPERTY] = \ + SUBNET_OPENSTACK_TYPE + ctx.instance.runtime_properties[OPENSTACK_NAME_PROPERTY] = subnet['name'] + + +@operation +@with_neutron_client +def delete(neutron_client, **kwargs): + delete_resource_and_runtime_properties(ctx, neutron_client, + RUNTIME_PROPERTIES_KEYS) + + +@operation +@with_neutron_client +def creation_validation(neutron_client, args, **kwargs): + validate_resource(ctx, neutron_client, SUBNET_OPENSTACK_TYPE) + subnet = dict(ctx.node.properties['subnet'], **args) + + if 'cidr' not in subnet: + err = '"cidr" property must appear under the "subnet" property of a ' \ + 'subnet node' + ctx.logger.error('VALIDATION ERROR: ' + err) + raise NonRecoverableError(err) + validate_ip_or_range_syntax(ctx, subnet['cidr']) diff --git a/aria/multivim-plugin/neutron_plugin/tests/__init__.py b/aria/multivim-plugin/neutron_plugin/tests/__init__.py new file mode 100644 index 0000000000..04cb21f745 --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/tests/__init__.py @@ -0,0 +1 @@ +__author__ = 'idanmo' diff --git a/aria/multivim-plugin/neutron_plugin/tests/test.py b/aria/multivim-plugin/neutron_plugin/tests/test.py new file mode 100644 index 0000000000..459c23a6cd --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/tests/test.py @@ -0,0 +1,220 @@ +######### +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +import mock +import random +import string +import unittest + +from cloudify.exceptions import NonRecoverableError +from cloudify.context import BootstrapContext + +from cloudify.mocks import MockCloudifyContext + +import openstack_plugin_common as common +import openstack_plugin_common.tests.test as common_test + +import neutron_plugin +import neutron_plugin.network +import neutron_plugin.port +import neutron_plugin.router +import neutron_plugin.security_group + + +class ResourcesRenamingTest(unittest.TestCase): + def setUp(self): + neutron_plugin.port._find_network_in_related_nodes = mock.Mock() + # *** Configs from files ******************** + common.Config.get = mock.Mock() + common.Config.get.return_value = {} + # *** Neutron ******************** + self.neutron_mock = mock.Mock() + + def neutron_mock_connect(unused_self, unused_cfg): + return self.neutron_mock + common.NeutronClient.connect = neutron_mock_connect + + self.neutron_mock.cosmo_list = mock.Mock() + self.neutron_mock.cosmo_list.return_value = [] + + def _setup_ctx(self, obj_type): + ctx = common_test.create_mock_ctx_with_provider_info( + node_id='__cloudify_id_something_001', + properties={ + obj_type: { + 'name': obj_type + '_name', + }, + 'rules': [] # For security_group + } + ) + return ctx + + def _test(self, obj_type): + ctx = self._setup_ctx(obj_type) + attr = getattr(self.neutron_mock, 'create_' + obj_type) + attr.return_value = { + obj_type: { + 'id': obj_type + '_id', + } + } + getattr(neutron_plugin, obj_type).create(ctx) + calls = attr.mock_calls + self.assertEquals(len(calls), 1) # Exactly one object created + # Indexes into call[]: + # 0 - the only call + # 1 - regular arguments + # 0 - first argument + arg = calls[0][1][0] + self.assertEquals(arg[obj_type]['name'], 'p2_' + obj_type + '_name') + + def test_network(self): + self._test('network') + + def test_port(self): + self._test('port') + + def test_router(self): + self._test('router') + + def test_security_group(self): + self._test('security_group') + + # Network chosen arbitrary for this test. + # Just testing something without prefix. + def test_network_no_prefix(self): + ctx = self._setup_ctx('network') + for pctx in common_test.BOOTSTRAP_CONTEXTS_WITHOUT_PREFIX: + ctx._bootstrap_context = BootstrapContext(pctx) + self.neutron_mock.create_network.reset_mock() + self.neutron_mock.create_network.return_value = { + 'network': { + 'id': 'network_id', + } + } + neutron_plugin.network.create(ctx) + calls = self.neutron_mock.create_network.mock_calls + self.assertEquals(len(calls), 1) # Exactly one network created + # Indexes into call[]: + # 0 - the only call + # 1 - regular arguments + # 0 - first argument + arg = calls[0][1][0] + self.assertEquals(arg['network']['name'], 'network_name', + "Failed with context: " + str(pctx)) + + +def _rand_str(n): + chars = string.ascii_uppercase + string.digits + return ''.join(random.choice(chars) for _ in range(n)) + + +class SecurityGroupTest(unittest.TestCase): + def setUp(self): + # *** Configs from files ******************** + common.Config.get = mock.Mock() + common.Config.get.return_value = {} + # *** Neutron ******************** + self.neutron_mock = mock.Mock() + + def neutron_mock_connect(unused_self, unused_cfg): + return self.neutron_mock + common.NeutronClient.connect = neutron_mock_connect + neutron_plugin.security_group._rules_for_sg_id = mock.Mock() + neutron_plugin.security_group._rules_for_sg_id.return_value = [] + + def _setup_ctx(self): + sg_name = _rand_str(6) + '_new' + ctx = MockCloudifyContext(properties={ + 'security_group': { + 'name': sg_name, + 'description': 'blah' + }, + 'rules': [{'port': 80}], + 'disable_default_egress_rules': True, + }) + return ctx + + def test_sg_new(self): + ctx = self._setup_ctx() + self.neutron_mock.cosmo_list = mock.Mock() + self.neutron_mock.cosmo_list.return_value = [] + self.neutron_mock.create_security_group = mock.Mock() + self.neutron_mock.create_security_group.return_value = { + 'security_group': { + 'description': 'blah', + 'id': ctx['security_group']['name'] + '_id', + } + } + neutron_plugin.security_group.create(ctx) + self.assertTrue(self.neutron_mock.create_security_group.mock_calls) + + def test_sg_use_existing(self): + ctx = self._setup_ctx() + self.neutron_mock.cosmo_list = mock.Mock() + self.neutron_mock.cosmo_list.return_value = [{ + 'id': ctx['security_group']['name'] + '_existing_id', + 'description': 'blah', + 'security_group_rules': [{ + 'remote_group_id': None, + 'direction': 'ingress', + 'protocol': 'tcp', + 'ethertype': 'IPv4', + 'port_range_max': 80, + 'port_range_min': 80, + 'remote_ip_prefix': '0.0.0.0/0', + }] + }] + self.neutron_mock.create_security_group = mock.Mock() + self.neutron_mock.create_security_group.return_value = { + 'security_group': { + 'description': 'blah', + 'id': ctx['security_group']['name'] + '_id', + } + } + neutron_plugin.security_group.create(ctx) + self.assertFalse(self.neutron_mock.create_security_group.mock_calls) + + def test_sg_use_existing_with_other_rules(self): + ctx = self._setup_ctx() + self.neutron_mock.cosmo_list = mock.Mock() + self.neutron_mock.cosmo_list.return_value = [{ + 'id': ctx['security_group']['name'] + '_existing_id', + 'description': 'blah', + 'security_group_rules': [{ + 'remote_group_id': None, + 'direction': 'ingress', + 'protocol': 'tcp', + 'ethertype': 'IPv4', + 'port_range_max': 81, # Note the different port! + 'port_range_min': 81, # Note the different port! + 'remote_ip_prefix': '0.0.0.0/0', + }] + }] + self.neutron_mock.create_security_group = mock.Mock() + self.neutron_mock.create_security_group.return_value = { + 'security_group': { + 'description': 'blah', + 'id': ctx['security_group']['name'] + '_id', + } + } + self.assertRaises( + NonRecoverableError, + neutron_plugin.security_group.create, + ctx + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/aria/multivim-plugin/neutron_plugin/tests/test_port.py b/aria/multivim-plugin/neutron_plugin/tests/test_port.py new file mode 100644 index 0000000000..1acee3d05d --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/tests/test_port.py @@ -0,0 +1,156 @@ +######## +# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +import unittest + +import mock + +import neutron_plugin.port +from cloudify.mocks import (MockCloudifyContext, + MockNodeInstanceContext, + MockRelationshipSubjectContext) +from openstack_plugin_common import (NeutronClientWithSugar, + OPENSTACK_ID_PROPERTY) +from cloudify.exceptions import OperationRetry + + +class TestPort(unittest.TestCase): + + def test_fixed_ips_no_fixed_ips(self): + node_props = {'fixed_ip': ''} + + with mock.patch( + 'neutron_plugin.port.' + 'get_openstack_id_of_single_connected_node_by_openstack_type', + self._get_connected_subnet_mock(return_empty=True)): + with mock.patch( + 'neutron_plugin.port.ctx', + self._get_mock_ctx_with_node_properties(node_props)): + + port = {} + neutron_plugin.port._handle_fixed_ips(port) + + self.assertNotIn('fixed_ips', port) + + def test_fixed_ips_subnet_only(self): + node_props = {'fixed_ip': ''} + + with mock.patch( + 'neutron_plugin.port.' + 'get_openstack_id_of_single_connected_node_by_openstack_type', + self._get_connected_subnet_mock(return_empty=False)): + with mock.patch( + 'neutron_plugin.port.ctx', + self._get_mock_ctx_with_node_properties(node_props)): + + port = {} + neutron_plugin.port._handle_fixed_ips(port) + + self.assertEquals([{'subnet_id': 'some-subnet-id'}], + port.get('fixed_ips')) + + def test_fixed_ips_ip_address_only(self): + node_props = {'fixed_ip': '1.2.3.4'} + + with mock.patch( + 'neutron_plugin.port.' + 'get_openstack_id_of_single_connected_node_by_openstack_type', + self._get_connected_subnet_mock(return_empty=True)): + with mock.patch( + 'neutron_plugin.port.ctx', + self._get_mock_ctx_with_node_properties(node_props)): + + port = {} + neutron_plugin.port._handle_fixed_ips(port) + + self.assertEquals([{'ip_address': '1.2.3.4'}], + port.get('fixed_ips')) + + def test_fixed_ips_subnet_and_ip_address(self): + node_props = {'fixed_ip': '1.2.3.4'} + + with mock.patch( + 'neutron_plugin.port.' + 'get_openstack_id_of_single_connected_node_by_openstack_type', + self._get_connected_subnet_mock(return_empty=False)): + with mock.patch( + 'neutron_plugin.port.ctx', + self._get_mock_ctx_with_node_properties(node_props)): + + port = {} + neutron_plugin.port._handle_fixed_ips(port) + + self.assertEquals([{'ip_address': '1.2.3.4', + 'subnet_id': 'some-subnet-id'}], + port.get('fixed_ips')) + + @staticmethod + def _get_connected_subnet_mock(return_empty=True): + return lambda *args, **kw: None if return_empty else 'some-subnet-id' + + @staticmethod + def _get_mock_ctx_with_node_properties(properties): + return MockCloudifyContext(node_id='test_node_id', + properties=properties) + + +class MockNeutronClient(NeutronClientWithSugar): + """A fake neutron client with hard-coded test data.""" + def __init__(self, update): + self.update = update + self.body = {'port': {'id': 'test-id', 'security_groups': []}} + + def show_port(self, *_): + return self.body + + def update_port(self, _, b, **__): + if self.update: + self.body.update(b) + return + + def cosmo_get(self, *_, **__): + return self.body['port'] + + +class TestPortSG(unittest.TestCase): + @mock.patch('openstack_plugin_common._put_client_in_kw') + def test_connect_sg_to_port(self, *_): + mock_neutron = MockNeutronClient(update=True) + ctx = MockCloudifyContext( + source=MockRelationshipSubjectContext(node=mock.MagicMock(), + instance=mock.MagicMock()), + target=MockRelationshipSubjectContext(node=mock.MagicMock(), + instance=mock.MagicMock())) + + with mock.patch('neutron_plugin.port.ctx', ctx): + neutron_plugin.port.connect_security_group(mock_neutron) + self.assertIsNone(ctx.operation._operation_retry) + + @mock.patch('openstack_plugin_common._put_client_in_kw') + def test_connect_sg_to_port_race_condition(self, *_): + mock_neutron = MockNeutronClient(update=False) + + ctx = MockCloudifyContext( + source=MockRelationshipSubjectContext(node=mock.MagicMock(), + instance=mock.MagicMock()), + target=MockRelationshipSubjectContext( + node=mock.MagicMock(), + instance=MockNodeInstanceContext( + runtime_properties={ + OPENSTACK_ID_PROPERTY: 'test-sg-id'}))) + with mock.patch('neutron_plugin.port.ctx', ctx): + neutron_plugin.port.connect_security_group(mock_neutron, ctx=ctx) + self.assertIsInstance(ctx.operation._operation_retry, + OperationRetry) diff --git a/aria/multivim-plugin/neutron_plugin/tests/test_security_group.py b/aria/multivim-plugin/neutron_plugin/tests/test_security_group.py new file mode 100644 index 0000000000..e958cddb33 --- /dev/null +++ b/aria/multivim-plugin/neutron_plugin/tests/test_security_group.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +######### +# Copyright (c) 2016 GigaSpaces Technologies Ltd. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. + +import unittest + +from mock import Mock, patch +from requests.exceptions import RequestException + +from neutron_plugin import security_group + +from cloudify.exceptions import NonRecoverableError +from cloudify.state import current_ctx + +from cloudify.mocks import MockCloudifyContext + + +class FakeException(Exception): + pass + + +@patch('openstack_plugin_common.OpenStackClient._validate_auth_params') +@patch('openstack_plugin_common.NeutronClientWithSugar') +class TestSecurityGroup(unittest.TestCase): + + def setUp(self): + super(TestSecurityGroup, self).setUp() + self.nova_client = Mock() + + self.ctx = MockCloudifyContext( + node_id='test', + deployment_id='test', + properties={ + 'description': 'The best Security Group. Great', + 'rules': [], + 'resource_id': 'mock_sg', + 'security_group': { + }, + 'server': {}, + 'openstack_config': { + 'auth_url': 'things/v3', + }, + }, + operation={'retry_number': 0}, + provider_context={'resources': {}} + ) + current_ctx.set(self.ctx) + self.addCleanup(current_ctx.clear) + + findctx = patch( + 'openstack_plugin_common._find_context_in_kw', + return_value=self.ctx, + ) + findctx.start() + self.addCleanup(findctx.stop) + + def test_set_sg_runtime_properties(self, mock_nc, *_): + security_group.create( + nova_client=self.nova_client, + ctx=self.ctx, + args={}, + ) + + self.assertEqual( + { + 'external_type': 'security_group', + 'external_id': mock_nc().get_id_from_resource(), + 'external_name': mock_nc().get_name_from_resource(), + }, + self.ctx.instance.runtime_properties + ) + + def test_create_sg_wait_timeout(self, mock_nc, *_): + mock_nc().show_security_group.side_effect = RequestException + + with self.assertRaises(NonRecoverableError): + security_group.create( + nova_client=self.nova_client, + ctx=self.ctx, + args={}, + status_attempts=3, + status_timeout=0.001, + ) + + @patch( + 'neutron_plugin.security_group.delete_resource_and_runtime_properties') + def test_dont_duplicate_if_failed_rule(self, mock_del_res, mock_nc, *_): + self.ctx.node.properties['rules'] = [ + { + 'port': '🍷', + }, + ] + mock_nc().create_security_group_rule.side_effect = FakeException + mock_del_res.side_effect = FakeException('the 2nd') + + with self.assertRaises(NonRecoverableError) as e: + security_group.create( + nova_client=self.nova_client, + ctx=self.ctx, + args={}, + ) + + self.assertIn('the 2nd', str(e.exception)) |