summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNate Potter <nathaniel.potteR@intel.com>2018-03-18 04:25:19 -0700
committerNate Potter <nathaniel.potteR@intel.com>2018-03-22 09:13:02 -0700
commit2a60a2c25b33a94d49859a83f0a9ce6127cdf452 (patch)
tree23743c278ef3990a1903a540ec0f71c5fb944724
parentce7349565ec25f0b0be99a558b927d45257cf1ae (diff)
Add HPA capability fetching to Newton
This patch adds functionality to get HPA features from OpenStack flavors and flavor extra specs when the AAI schema is versioned to support them. Change-Id: I6b14a72867fea86922244e974f92383e03edce98 Signed-off-by: Nathaniel Potter <nathaniel.potter@intel.com> Issue-ID: MULTICLOUD-179
-rw-r--r--newton/newton/registration/tests/test_registration.py169
-rw-r--r--newton/newton/registration/views/hpa.json129
-rw-r--r--newton/newton/registration/views/registration.py182
3 files changed, 472 insertions, 8 deletions
diff --git a/newton/newton/registration/tests/test_registration.py b/newton/newton/registration/tests/test_registration.py
index 78d75341..0e9be29f 100644
--- a/newton/newton/registration/tests/test_registration.py
+++ b/newton/newton/registration/tests/test_registration.py
@@ -14,9 +14,11 @@
import mock
+from django.conf import settings
from rest_framework import status
from common.utils import restcall
+from newton_base.openoapi.flavor import Flavors
from newton_base.tests import mock_info
from newton_base.tests import test_base
from newton_base.util import VimDriverUtils
@@ -45,6 +47,149 @@ MOCK_GET_FLAVOR_RESPONSE = {
]
}
+MOCK_GET_EXTRA_SPECS_RESPONSE = {
+ "extra_specs": {
+ "hw:cpu_sockets": 4,
+ "hw:cpu_cores": 4,
+ "hw:cpu_policy": "dedicated",
+ "hw:numa_nodes": 3,
+ "hw:numa_cpus.1": [0, 1],
+ "hw:numa_mem.1": 2,
+ "pci_passthrough:alias": "mycrypto-8086-0443:4",
+ "hw:mem_page_size": "1GB"
+ }
+}
+
+MOCK_HPA_RESPONSE = """{
+ "basicCapabilities": {
+ "info": {
+ "hpa-feature": "basicCapabilities",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "vcpus": {
+ "key": "numVirtualCpu",
+ "unit": null
+ },
+ "ram": {
+ "key": "virtualMemSize",
+ "unit": "GB"
+ }
+ }
+ },
+ "localStorage": {
+ "info": {
+ "hpa-feature": "localStorage",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "disk": {
+ "key": "diskSize",
+ "unit": "GB"
+ },
+ "swap": {
+ "key": "swapMemSize",
+ "unit": "MB"
+ }
+ }
+ },
+ "cpuTopology": {
+ "info": {
+ "hpa-feature": "cpuTopology",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:cpu_sockets": {
+ "key": "numCpuSockets",
+ "unit": null
+ },
+ "hw:cpu_cores": {
+ "key": "numCpuCores",
+ "unit": null
+ },
+ "hw:cpu_threads": {
+ "key": "numCpuThreads",
+ "unit": null
+ }
+ }
+ },
+ "cpuPinning": {
+ "info": {
+ "hpa-feature": "cpuPinning",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:cpu_thread_policy": {
+ "key": "logicalCpuThreadPinningPolicy",
+ "unit": null
+ },
+ "hw:cpu_policy": {
+ "key": "logicalCpuPinningPolicy",
+ "unit": null
+ }
+ }
+ },
+ "numa": {
+ "info": {
+ "hpa-feature": "numa",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:numa_nodes": {
+ "key": "numaNodes",
+ "unit": null
+ },
+ "hw:numa_cpus": {
+ "key": "numaCpu",
+ "unit": null
+ },
+ "hw:numa_mem": {
+ "key": "numaMem",
+ "unit": "GB"
+ }
+ }
+ },
+ "hugePages": {
+ "info": {
+ "hpa-feature": "hugePages",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:mem_page_size": {
+ "key": "memoryPageSize",
+ "unit": null
+ }
+ }
+ },
+ "pciePassthrough": {
+ "info": {
+ "hpa-feature": "pciePassthrough",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "pci_count": {
+ "key": "pciCount",
+ "unit": null
+ },
+ "pci_vendor_id": {
+ "key": "pciVendorId",
+ "unit": null
+ },
+ "pci_device_id": {
+ "key": "pciDeviceId",
+ "unit": null
+ }
+ }
+ }
+}"""
+
MOCK_GET_IMAGE_RESPONSE = {
"images": [
{
@@ -135,10 +280,13 @@ class TestFlavors(test_base.TestRequest):
mock_response.json.return_value = return_value
return mock_response
+ @mock.patch.object(Flavors, '_get_flavor_extra_specs')
@mock.patch.object(VimDriverUtils, 'get_session')
@mock.patch.object(VimDriverUtils, 'get_vim_info')
def test_register_endpoint_successfully(
- self, mock_get_vim_info, mock_get_session):
+ self, mock_get_vim_info, mock_get_session,
+ mock_get_extra_specs):
+ settings.AAI_SCHEMA_VERSION = "v13"
restcall.req_to_aai = mock.Mock()
restcall.req_to_aai.return_value = (0, {}, status.HTTP_200_OK)
mock_get_vim_info.return_value = mock_info.MOCK_VIM_INFO
@@ -157,15 +305,20 @@ class TestFlavors(test_base.TestRequest):
MOCK_GET_HYPERVISOR_RESPONSE)
]
})
+ mock_extra_specs_response = mock.Mock(spec=test_base.MockResponse)
+ mock_extra_specs_response.status_code = status.HTTP_200_OK
+ mock_extra_specs_response.json.return_value = MOCK_GET_EXTRA_SPECS_RESPONSE
+ mock_get_extra_specs.return_value = mock_extra_specs_response
- response = self.client.post((
- "/api/%s/v0/windriver-hudson-dc_RegionOne/"
- "registry" % test_base.MULTIVIM_VERSION),
- TEST_REGISTER_ENDPOINT_REQUEST,
- HTTP_X_AUTH_TOKEN=mock_info.MOCK_TOKEN_ID)
+ with mock.patch('__builtin__.open', mock.mock_open(read_data=MOCK_HPA_RESPONSE)) as mock_file:
+ response = self.client.post((
+ "/api/%s/v0/windriver-hudson-dc_RegionOne/"
+ "registry" % test_base.MULTIVIM_VERSION),
+ TEST_REGISTER_ENDPOINT_REQUEST,
+ HTTP_X_AUTH_TOKEN=mock_info.MOCK_TOKEN_ID)
- self.assertEquals(status.HTTP_202_ACCEPTED,
- response.status_code)
+ self.assertEquals(status.HTTP_202_ACCEPTED,
+ response.status_code)
@mock.patch.object(VimDriverUtils, 'delete_vim_info')
def test_unregister_endpoint_successfully(
diff --git a/newton/newton/registration/views/hpa.json b/newton/newton/registration/views/hpa.json
new file mode 100644
index 00000000..832d55af
--- /dev/null
+++ b/newton/newton/registration/views/hpa.json
@@ -0,0 +1,129 @@
+{
+ "basicCapabilities": {
+ "info": {
+ "hpa-feature": "basicCapabilities",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "vcpus": {
+ "key": "numVirtualCpu",
+ "unit": null
+ },
+ "ram": {
+ "key": "virtualMemSize",
+ "unit": "GB"
+ }
+ }
+ },
+ "localStorage": {
+ "info": {
+ "hpa-feature": "localStorage",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "disk": {
+ "key": "diskSize",
+ "unit": "GB"
+ },
+ "swap": {
+ "key": "swapMemSize",
+ "unit": "MB"
+ }
+ }
+ },
+ "cpuTopology": {
+ "info": {
+ "hpa-feature": "cpuTopology",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:cpu_sockets": {
+ "key": "numCpuSockets",
+ "unit": null
+ },
+ "hw:cpu_cores": {
+ "key": "numCpuCores",
+ "unit": null
+ },
+ "hw:cpu_threads": {
+ "key": "numCpuThreads",
+ "unit": null
+ }
+ }
+ },
+ "cpuPinning": {
+ "info": {
+ "hpa-feature": "cpuPinning",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:cpu_thread_policy": {
+ "key": "logicalCpuThreadPinningPolicy",
+ "unit": null
+ },
+ "hw:cpu_policy": {
+ "key": "logicalCpuPinningPolicy",
+ "unit": null
+ }
+ }
+ },
+ "numa": {
+ "info": {
+ "hpa-feature": "numa",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:numa_nodes": {
+ "key": "numaNodes",
+ "unit": null
+ },
+ "hw:numa_cpus": {
+ "key": "numaCpu",
+ "unit": null
+ },
+ "hw:numa_mem": {
+ "key": "numaMem",
+ "unit": "GB"
+ }
+ }
+ },
+ "hugePages": {
+ "info": {
+ "hpa-feature": "hugePages",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "hw:mem_page_size": {
+ "key": "memoryPageSize",
+ "unit": null
+ }
+ }
+ },
+ "pciePassthrough": {
+ "info": {
+ "hpa-feature": "pciePassthrough",
+ "hpa-version": "v1",
+ "architecture": "generic"
+ },
+ "hpa-attributes": {
+ "pci_count": {
+ "key": "pciCount",
+ "unit": null
+ },
+ "pci_vendor_id": {
+ "key": "pciVendorId",
+ "unit": null
+ },
+ "pci_device_id": {
+ "key": "pciDeviceId",
+ "unit": null
+ }
+ }
+ }
+}
diff --git a/newton/newton/registration/views/registration.py b/newton/newton/registration/views/registration.py
index 69c85837..e9f1a357 100644
--- a/newton/newton/registration/views/registration.py
+++ b/newton/newton/registration/views/registration.py
@@ -13,10 +13,16 @@
# limitations under the License.
import logging
+import json
+import uuid
from django.conf import settings
+from keystoneauth1.exceptions import HttpError
+from common.exceptions import VimDriverNewtonException
+from common.msapi import extsys
from newton_base.registration import registration as newton_registration
+from newton_base.openoapi.flavor import Flavors
logger = logging.getLogger(__name__)
@@ -28,3 +34,179 @@ class Registry(newton_registration.Registry):
self.proxy_prefix = settings.MULTICLOUD_PREFIX
self.aai_base_url = settings.AAI_BASE_URL
self._logger = logger
+
+ def _discover_flavors(self, vimid="", session=None, viminfo=None):
+ try:
+ cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid)
+ for flavor in self._get_list_resources(
+ "/flavors/detail", "compute", session, viminfo, vimid,
+ "flavors"):
+ flavor_info = {
+ 'flavor-id': flavor['id'],
+ 'flavor-name': flavor['name'],
+ 'flavor-vcpus': flavor['vcpus'],
+ 'flavor-ram': flavor['ram'],
+ 'flavor-disk': flavor['disk'],
+ 'flavor-ephemeral': flavor['OS-FLV-EXT-DATA:ephemeral'],
+ 'flavor-swap': flavor['swap'],
+ 'flavor-is-public': flavor['os-flavor-access:is_public'],
+ 'flavor-disabled': flavor['OS-FLV-DISABLED:disabled'],
+ }
+ if flavor.get('link') and len(flavor['link']) > 0:
+ flavor_info['flavor-selflink'] = flavor['link'][0]['href'] or 'http://0.0.0.0',
+ else:
+ flavor_info['flavor-selflink'] = 'http://0.0.0.0',
+
+ if settings.AAI_SCHEMA_VERSION == "v13":
+ extraResp = Flavors._get_flavor_extra_specs(session, flavor['id'])
+ extraContent = extraResp.json()
+ hpa_capabilities = self._get_hpa_capabilities(vimid, flavor,
+ extraContent["extra_specs"])
+ flavor_info['hpa_capabilities'] = hpa_capabilities
+
+ self._update_resoure(
+ cloud_owner, cloud_region_id, flavor['id'],
+ flavor_info, "flavor")
+
+ except VimDriverNewtonException as e:
+ self._logger.error("VimDriverNewtonException: status:%s, response:%s" % (e.http_status, e.content))
+ return
+ except HttpError as e:
+ self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+ return
+ except Exception as e:
+ self._logger.error(traceback.format_exc())
+ return
+
+ def _get_hpa_capabilities(self, vimid, flavor, extra_specs):
+ """Convert flavor information to HPA capabilities for AAI"""
+ cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid)
+
+ json_data = open('hpa.json').read()
+ hpa_dict = json.loads(json_data)
+
+ capabilities = []
+
+ # Basic Capabilities
+ if set(hpa_dict['basicCapabilities']['hpa-attributes']).intersection(flavor):
+ capability = hpa_dict['basicCapabilities']['info']
+ capability['hpa-capability-id'] = str(uuid.uuid4())
+ capability['hpa-feature-attributes'] = self._get_capability_attributes(
+ flavor,
+ hpa_dict['basicCapabilities']['hpa-attributes'])
+ capabilities.append(capability)
+
+ # Local Storage
+ if set(hpa_dict['localStorage']['hpa-attributes']).intersection(flavor):
+ capability = hpa_dict['localStorage']['info']
+ capability['hpa-capability-id'] = str(uuid.uuid4())
+ capability['hpa-feature-attributes'] = self._get_capability_attributes(
+ flavor,
+ hpa_dict['localStorage']['hpa-attributes'])
+ capabilities.append(capability)
+
+ # CPU Topology
+ if set(hpa_dict['cpuTopology']['hpa-attributes']).intersection(extra_specs):
+ capability = hpa_dict['cpuTopology']['info']
+ capability['hpa-capability-id'] = str(uuid.uuid4())
+ capability['hpa-features-attributes'] = self._get_capability_attributes(
+ extra_specs,
+ hpa_dict['cpuTopology']['hpa-attributes'])
+ capabilities.append(capability)
+
+ # CPU Pinning
+ if set(hpa_dict['cpuPinning']['hpa-attributes']).intersection(extra_specs):
+ capability = hpa_dict['cpuPinning']['info']
+ capability['hpa-capability-id'] = str(uuid.uuid4())
+ capability['hpa-features-attributes'] = self._get_capability_attributes(
+ extra_specs,
+ hpa_dict['cpuPinning']['hpa-attributes'])
+ capabilities.append(capability)
+
+ # Huge Pages
+ if set(hpa_dict['hugePages']['hpa-attributes']).intersection(extra_specs):
+ capability = hpa_dict['hugePages']['info']
+ if extra_specs['hw:mem_page_size'] not in ['small', 'large', 'any']:
+ unit = ''.join(i for i in extra_specs['hw:mem_page_size'] if not i.isdigit())
+ if unit == '':
+ unit = 'KB'
+ hpa_dict['hugePages']['hpa-attributes']['hw:mem_page_size']['unit'] = unit
+ capability['hpa-capability-id'] = str(uuid.uuid4())
+ capability['hpa-features-attributes'] = self._get_capability_attributes(
+ extra_specs,
+ hpa_dict['hugePages']['hpa-attributes'])
+ capabilities.append(capability)
+
+ # NUMA
+ if "hw:numa_nodes" in extra_specs:
+ capability = hpa_dict['numa']['info']
+ capability['hpa-capability-id'] = str(uuid.uuid4())
+ # NUMA nodes are a special case and can't use the generic get attrs function
+ attributes = []
+ attributes.append({
+ 'hpa-attribute-key': hpa_dict['numa']['hpa-attributes']['hw:numa_nodes']['key'],
+ 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(extra_specs['hw:numa_nodes'])
+ })
+ for spec in extra_specs:
+ if spec.startswith('hw:numa_cpus'):
+ cpu_num = spec.split(":")[-1]
+ attributes.append({
+ 'hpa-attribute-key': 'numaCpu-' + cpu_num,
+ 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(extra_specs[spec])
+ })
+ elif spec.startswith('hw:numa_mem'):
+ mem_num = spec.split(":")[-1]
+ attributes.append({
+ 'hpa-attribute-key': 'numaMem-' + mem_num,
+ 'hpa-attribute-value': '{{\"value\":\"{0}\",\"unit\":\"{1}\"}}'.format(extra_specs[spec],
+ "GB")
+ })
+ capability['hpa-features-attributes'] = attributes
+ capabilities.append(capability)
+
+ # PCIe Passthrough
+ pci_devices = [spec for spec in extra_specs if spec.startswith("pci_passthrough:alias")]
+ for device in pci_devices:
+ capability = hpa_dict['pciePassthrough']['info']
+ capability['hpa-capability-id'] = str(uuid.uuid4())
+ # device will be in the form pci_passthrough:alias=ALIAS:COUNT,
+ # ALIAS is expected to be in the form <NAME>-<VENDOR_ID>-<DEVICE_ID>
+ device_info = extra_specs[device].split(":")
+ count = device_info[-1]
+ vendor_id = device_info[0].split("-")[1]
+ device_id = device_info[0].split("-")[2]
+
+ attributes = [
+ {
+ 'hpa-attribute-key': 'pciCount',
+ 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(count)
+ },
+ {
+ 'hpa-attribute-key': 'pciVendorId',
+ 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(vendor_id)
+ },
+ {
+ 'hpa-attribute-key': 'pciDeviceId',
+ 'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(device_id)
+ }
+ ]
+
+ capability['hpa-features-attributes'] = attributes
+ capabilities.append(capability)
+
+ return capabilities
+
+ def _get_capability_attributes(self, cloud_info, attributes):
+ result = []
+ for attr in attributes:
+ if attr in cloud_info:
+ attribute = {'hpa-attribute-key': attributes[attr]['key']}
+ if attributes[attr]['unit']:
+ attribute['hpa-attribute-value'] = (
+ '{{\"value\":\"{0}\",\"unit\":\"{1}\"}}').format(cloud_info[attr],
+ attributes[attr]['unit'])
+ else:
+ attribute['hpa-attribute-value'] = '{{\"value\":\"{0}\"}}'.format(cloud_info[attr])
+
+ result.append(attribute)
+ return result