From 2a60a2c25b33a94d49859a83f0a9ce6127cdf452 Mon Sep 17 00:00:00 2001 From: Nate Potter Date: Sun, 18 Mar 2018 04:25:19 -0700 Subject: Add HPA capability fetching to Newton This patch adds functionality to get HPA features from OpenStack flavors and flavor extra specs when the AAI schema is versioned to support them. Change-Id: I6b14a72867fea86922244e974f92383e03edce98 Signed-off-by: Nathaniel Potter Issue-ID: MULTICLOUD-179 --- .../newton/registration/tests/test_registration.py | 169 ++++++++++++++++++- newton/newton/registration/views/hpa.json | 129 +++++++++++++++ newton/newton/registration/views/registration.py | 182 +++++++++++++++++++++ 3 files changed, 472 insertions(+), 8 deletions(-) create mode 100644 newton/newton/registration/views/hpa.json (limited to 'newton') diff --git a/newton/newton/registration/tests/test_registration.py b/newton/newton/registration/tests/test_registration.py index 78d75341..0e9be29f 100644 --- a/newton/newton/registration/tests/test_registration.py +++ b/newton/newton/registration/tests/test_registration.py @@ -14,9 +14,11 @@ import mock +from django.conf import settings from rest_framework import status from common.utils import restcall +from newton_base.openoapi.flavor import Flavors from newton_base.tests import mock_info from newton_base.tests import test_base from newton_base.util import VimDriverUtils @@ -45,6 +47,149 @@ MOCK_GET_FLAVOR_RESPONSE = { ] } +MOCK_GET_EXTRA_SPECS_RESPONSE = { + "extra_specs": { + "hw:cpu_sockets": 4, + "hw:cpu_cores": 4, + "hw:cpu_policy": "dedicated", + "hw:numa_nodes": 3, + "hw:numa_cpus.1": [0, 1], + "hw:numa_mem.1": 2, + "pci_passthrough:alias": "mycrypto-8086-0443:4", + "hw:mem_page_size": "1GB" + } +} + +MOCK_HPA_RESPONSE = """{ + "basicCapabilities": { + "info": { + "hpa-feature": "basicCapabilities", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "vcpus": { + "key": "numVirtualCpu", + "unit": null + }, + "ram": { + "key": "virtualMemSize", + "unit": "GB" + } + } + }, + "localStorage": { + "info": { + "hpa-feature": "localStorage", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "disk": { + "key": "diskSize", + "unit": "GB" + }, + "swap": { + "key": "swapMemSize", + "unit": "MB" + } + } + }, + "cpuTopology": { + "info": { + "hpa-feature": "cpuTopology", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:cpu_sockets": { + "key": "numCpuSockets", + "unit": null + }, + "hw:cpu_cores": { + "key": "numCpuCores", + "unit": null + }, + "hw:cpu_threads": { + "key": "numCpuThreads", + "unit": null + } + } + }, + "cpuPinning": { + "info": { + "hpa-feature": "cpuPinning", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:cpu_thread_policy": { + "key": "logicalCpuThreadPinningPolicy", + "unit": null + }, + "hw:cpu_policy": { + "key": "logicalCpuPinningPolicy", + "unit": null + } + } + }, + "numa": { + "info": { + "hpa-feature": "numa", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:numa_nodes": { + "key": "numaNodes", + "unit": null + }, + "hw:numa_cpus": { + "key": "numaCpu", + "unit": null + }, + "hw:numa_mem": { + "key": "numaMem", + "unit": "GB" + } + } + }, + "hugePages": { + "info": { + "hpa-feature": "hugePages", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:mem_page_size": { + "key": "memoryPageSize", + "unit": null + } + } + }, + "pciePassthrough": { + "info": { + "hpa-feature": "pciePassthrough", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "pci_count": { + "key": "pciCount", + "unit": null + }, + "pci_vendor_id": { + "key": "pciVendorId", + "unit": null + }, + "pci_device_id": { + "key": "pciDeviceId", + "unit": null + } + } + } +}""" + MOCK_GET_IMAGE_RESPONSE = { "images": [ { @@ -135,10 +280,13 @@ class TestFlavors(test_base.TestRequest): mock_response.json.return_value = return_value return mock_response + @mock.patch.object(Flavors, '_get_flavor_extra_specs') @mock.patch.object(VimDriverUtils, 'get_session') @mock.patch.object(VimDriverUtils, 'get_vim_info') def test_register_endpoint_successfully( - self, mock_get_vim_info, mock_get_session): + self, mock_get_vim_info, mock_get_session, + mock_get_extra_specs): + settings.AAI_SCHEMA_VERSION = "v13" restcall.req_to_aai = mock.Mock() restcall.req_to_aai.return_value = (0, {}, status.HTTP_200_OK) mock_get_vim_info.return_value = mock_info.MOCK_VIM_INFO @@ -157,15 +305,20 @@ class TestFlavors(test_base.TestRequest): MOCK_GET_HYPERVISOR_RESPONSE) ] }) + mock_extra_specs_response = mock.Mock(spec=test_base.MockResponse) + mock_extra_specs_response.status_code = status.HTTP_200_OK + mock_extra_specs_response.json.return_value = MOCK_GET_EXTRA_SPECS_RESPONSE + mock_get_extra_specs.return_value = mock_extra_specs_response - response = self.client.post(( - "/api/%s/v0/windriver-hudson-dc_RegionOne/" - "registry" % test_base.MULTIVIM_VERSION), - TEST_REGISTER_ENDPOINT_REQUEST, - HTTP_X_AUTH_TOKEN=mock_info.MOCK_TOKEN_ID) + with mock.patch('__builtin__.open', mock.mock_open(read_data=MOCK_HPA_RESPONSE)) as mock_file: + response = self.client.post(( + "/api/%s/v0/windriver-hudson-dc_RegionOne/" + "registry" % test_base.MULTIVIM_VERSION), + TEST_REGISTER_ENDPOINT_REQUEST, + HTTP_X_AUTH_TOKEN=mock_info.MOCK_TOKEN_ID) - self.assertEquals(status.HTTP_202_ACCEPTED, - response.status_code) + self.assertEquals(status.HTTP_202_ACCEPTED, + response.status_code) @mock.patch.object(VimDriverUtils, 'delete_vim_info') def test_unregister_endpoint_successfully( diff --git a/newton/newton/registration/views/hpa.json b/newton/newton/registration/views/hpa.json new file mode 100644 index 00000000..832d55af --- /dev/null +++ b/newton/newton/registration/views/hpa.json @@ -0,0 +1,129 @@ +{ + "basicCapabilities": { + "info": { + "hpa-feature": "basicCapabilities", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "vcpus": { + "key": "numVirtualCpu", + "unit": null + }, + "ram": { + "key": "virtualMemSize", + "unit": "GB" + } + } + }, + "localStorage": { + "info": { + "hpa-feature": "localStorage", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "disk": { + "key": "diskSize", + "unit": "GB" + }, + "swap": { + "key": "swapMemSize", + "unit": "MB" + } + } + }, + "cpuTopology": { + "info": { + "hpa-feature": "cpuTopology", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:cpu_sockets": { + "key": "numCpuSockets", + "unit": null + }, + "hw:cpu_cores": { + "key": "numCpuCores", + "unit": null + }, + "hw:cpu_threads": { + "key": "numCpuThreads", + "unit": null + } + } + }, + "cpuPinning": { + "info": { + "hpa-feature": "cpuPinning", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:cpu_thread_policy": { + "key": "logicalCpuThreadPinningPolicy", + "unit": null + }, + "hw:cpu_policy": { + "key": "logicalCpuPinningPolicy", + "unit": null + } + } + }, + "numa": { + "info": { + "hpa-feature": "numa", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:numa_nodes": { + "key": "numaNodes", + "unit": null + }, + "hw:numa_cpus": { + "key": "numaCpu", + "unit": null + }, + "hw:numa_mem": { + "key": "numaMem", + "unit": "GB" + } + } + }, + "hugePages": { + "info": { + "hpa-feature": "hugePages", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "hw:mem_page_size": { + "key": "memoryPageSize", + "unit": null + } + } + }, + "pciePassthrough": { + "info": { + "hpa-feature": "pciePassthrough", + "hpa-version": "v1", + "architecture": "generic" + }, + "hpa-attributes": { + "pci_count": { + "key": "pciCount", + "unit": null + }, + "pci_vendor_id": { + "key": "pciVendorId", + "unit": null + }, + "pci_device_id": { + "key": "pciDeviceId", + "unit": null + } + } + } +} diff --git a/newton/newton/registration/views/registration.py b/newton/newton/registration/views/registration.py index 69c85837..e9f1a357 100644 --- a/newton/newton/registration/views/registration.py +++ b/newton/newton/registration/views/registration.py @@ -13,10 +13,16 @@ # limitations under the License. import logging +import json +import uuid from django.conf import settings +from keystoneauth1.exceptions import HttpError +from common.exceptions import VimDriverNewtonException +from common.msapi import extsys from newton_base.registration import registration as newton_registration +from newton_base.openoapi.flavor import Flavors logger = logging.getLogger(__name__) @@ -28,3 +34,179 @@ class Registry(newton_registration.Registry): self.proxy_prefix = settings.MULTICLOUD_PREFIX self.aai_base_url = settings.AAI_BASE_URL self._logger = logger + + def _discover_flavors(self, vimid="", session=None, viminfo=None): + try: + cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) + for flavor in self._get_list_resources( + "/flavors/detail", "compute", session, viminfo, vimid, + "flavors"): + flavor_info = { + 'flavor-id': flavor['id'], + 'flavor-name': flavor['name'], + 'flavor-vcpus': flavor['vcpus'], + 'flavor-ram': flavor['ram'], + 'flavor-disk': flavor['disk'], + 'flavor-ephemeral': flavor['OS-FLV-EXT-DATA:ephemeral'], + 'flavor-swap': flavor['swap'], + 'flavor-is-public': flavor['os-flavor-access:is_public'], + 'flavor-disabled': flavor['OS-FLV-DISABLED:disabled'], + } + if flavor.get('link') and len(flavor['link']) > 0: + flavor_info['flavor-selflink'] = flavor['link'][0]['href'] or 'http://0.0.0.0', + else: + flavor_info['flavor-selflink'] = 'http://0.0.0.0', + + if settings.AAI_SCHEMA_VERSION == "v13": + extraResp = Flavors._get_flavor_extra_specs(session, flavor['id']) + extraContent = extraResp.json() + hpa_capabilities = self._get_hpa_capabilities(vimid, flavor, + extraContent["extra_specs"]) + flavor_info['hpa_capabilities'] = hpa_capabilities + + self._update_resoure( + cloud_owner, cloud_region_id, flavor['id'], + flavor_info, "flavor") + + except VimDriverNewtonException as e: + self._logger.error("VimDriverNewtonException: status:%s, response:%s" % (e.http_status, e.content)) + return + except HttpError as e: + self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json())) + return + except Exception as e: + self._logger.error(traceback.format_exc()) + return + + def _get_hpa_capabilities(self, vimid, flavor, extra_specs): + """Convert flavor information to HPA capabilities for AAI""" + cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid) + + json_data = open('hpa.json').read() + hpa_dict = json.loads(json_data) + + capabilities = [] + + # Basic Capabilities + if set(hpa_dict['basicCapabilities']['hpa-attributes']).intersection(flavor): + capability = hpa_dict['basicCapabilities']['info'] + capability['hpa-capability-id'] = str(uuid.uuid4()) + capability['hpa-feature-attributes'] = self._get_capability_attributes( + flavor, + hpa_dict['basicCapabilities']['hpa-attributes']) + capabilities.append(capability) + + # Local Storage + if set(hpa_dict['localStorage']['hpa-attributes']).intersection(flavor): + capability = hpa_dict['localStorage']['info'] + capability['hpa-capability-id'] = str(uuid.uuid4()) + capability['hpa-feature-attributes'] = self._get_capability_attributes( + flavor, + hpa_dict['localStorage']['hpa-attributes']) + capabilities.append(capability) + + # CPU Topology + if set(hpa_dict['cpuTopology']['hpa-attributes']).intersection(extra_specs): + capability = hpa_dict['cpuTopology']['info'] + capability['hpa-capability-id'] = str(uuid.uuid4()) + capability['hpa-features-attributes'] = self._get_capability_attributes( + extra_specs, + hpa_dict['cpuTopology']['hpa-attributes']) + capabilities.append(capability) + + # CPU Pinning + if set(hpa_dict['cpuPinning']['hpa-attributes']).intersection(extra_specs): + capability = hpa_dict['cpuPinning']['info'] + capability['hpa-capability-id'] = str(uuid.uuid4()) + capability['hpa-features-attributes'] = self._get_capability_attributes( + extra_specs, + hpa_dict['cpuPinning']['hpa-attributes']) + capabilities.append(capability) + + # Huge Pages + if set(hpa_dict['hugePages']['hpa-attributes']).intersection(extra_specs): + capability = hpa_dict['hugePages']['info'] + if extra_specs['hw:mem_page_size'] not in ['small', 'large', 'any']: + unit = ''.join(i for i in extra_specs['hw:mem_page_size'] if not i.isdigit()) + if unit == '': + unit = 'KB' + hpa_dict['hugePages']['hpa-attributes']['hw:mem_page_size']['unit'] = unit + capability['hpa-capability-id'] = str(uuid.uuid4()) + capability['hpa-features-attributes'] = self._get_capability_attributes( + extra_specs, + hpa_dict['hugePages']['hpa-attributes']) + capabilities.append(capability) + + # NUMA + if "hw:numa_nodes" in extra_specs: + capability = hpa_dict['numa']['info'] + capability['hpa-capability-id'] = str(uuid.uuid4()) + # NUMA nodes are a special case and can't use the generic get attrs function + attributes = [] + attributes.append({ + 'hpa-attribute-key': hpa_dict['numa']['hpa-attributes']['hw:numa_nodes']['key'], + 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(extra_specs['hw:numa_nodes']) + }) + for spec in extra_specs: + if spec.startswith('hw:numa_cpus'): + cpu_num = spec.split(":")[-1] + attributes.append({ + 'hpa-attribute-key': 'numaCpu-' + cpu_num, + 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(extra_specs[spec]) + }) + elif spec.startswith('hw:numa_mem'): + mem_num = spec.split(":")[-1] + attributes.append({ + 'hpa-attribute-key': 'numaMem-' + mem_num, + 'hpa-attribute-value': '{{\"value\":\"{0}\",\"unit\":\"{1}\"}}'.format(extra_specs[spec], + "GB") + }) + capability['hpa-features-attributes'] = attributes + capabilities.append(capability) + + # PCIe Passthrough + pci_devices = [spec for spec in extra_specs if spec.startswith("pci_passthrough:alias")] + for device in pci_devices: + capability = hpa_dict['pciePassthrough']['info'] + capability['hpa-capability-id'] = str(uuid.uuid4()) + # device will be in the form pci_passthrough:alias=ALIAS:COUNT, + # ALIAS is expected to be in the form -- + device_info = extra_specs[device].split(":") + count = device_info[-1] + vendor_id = device_info[0].split("-")[1] + device_id = device_info[0].split("-")[2] + + attributes = [ + { + 'hpa-attribute-key': 'pciCount', + 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(count) + }, + { + 'hpa-attribute-key': 'pciVendorId', + 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(vendor_id) + }, + { + 'hpa-attribute-key': 'pciDeviceId', + 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(device_id) + } + ] + + capability['hpa-features-attributes'] = attributes + capabilities.append(capability) + + return capabilities + + def _get_capability_attributes(self, cloud_info, attributes): + result = [] + for attr in attributes: + if attr in cloud_info: + attribute = {'hpa-attribute-key': attributes[attr]['key']} + if attributes[attr]['unit']: + attribute['hpa-attribute-value'] = ( + '{{\"value\":\"{0}\",\"unit\":\"{1}\"}}').format(cloud_info[attr], + attributes[attr]['unit']) + else: + attribute['hpa-attribute-value'] = '{{\"value\":\"{0}\"}}'.format(cloud_info[attr]) + + result.append(attribute) + return result -- cgit 1.2.3-korg