Add HPA capability fetching to Newton

This patch adds functionality to get HPA features from
OpenStack flavors and flavor extra specs when the AAI
schema is versioned to support them.

Change-Id: I6b14a72867fea86922244e974f92383e03edce98
Signed-off-by: Nathaniel Potter <nathaniel.potter@intel.com>
Issue-ID: MULTICLOUD-179

3 files changed, 472 insertions, 8 deletions

diff --git a/newton/newton/registration/tests/test_registration.py b/newton/newton/registration/tests/test_registration.py
index 78d75341..0e9be29f 100644
--- a/newton/newton/registration/tests/test_registration.py
+++ b/newton/newton/registration/tests/test_registration.py
@@ -14,9 +14,11 @@
 
 import mock
 
+from django.conf import settings
 from rest_framework import status
 
 from common.utils import restcall
+from newton_base.openoapi.flavor import Flavors
 from newton_base.tests import mock_info
 from newton_base.tests import test_base
 from newton_base.util import VimDriverUtils
@@ -45,6 +47,149 @@ MOCK_GET_FLAVOR_RESPONSE = {
     ]
 }
 
+MOCK_GET_EXTRA_SPECS_RESPONSE = {
+    "extra_specs": {
+        "hw:cpu_sockets": 4,
+        "hw:cpu_cores": 4,
+        "hw:cpu_policy": "dedicated",
+        "hw:numa_nodes": 3,
+        "hw:numa_cpus.1": [0, 1],
+        "hw:numa_mem.1": 2,
+        "pci_passthrough:alias": "mycrypto-8086-0443:4",
+        "hw:mem_page_size": "1GB"
+    }
+}
+
+MOCK_HPA_RESPONSE = """{
+    "basicCapabilities": {
+        "info": {
+            "hpa-feature": "basicCapabilities",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "vcpus": {
+                "key": "numVirtualCpu",
+                "unit": null
+            },
+            "ram": {
+                "key": "virtualMemSize",
+                "unit": "GB"
+            }
+        }
+    },
+    "localStorage": {
+        "info": {
+            "hpa-feature": "localStorage",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "disk": {
+                "key": "diskSize",
+                "unit": "GB"
+            },
+            "swap": {
+                "key": "swapMemSize",
+                "unit": "MB"
+            }
+        }
+    },
+    "cpuTopology": {
+        "info": {
+            "hpa-feature": "cpuTopology",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:cpu_sockets": {
+                "key": "numCpuSockets",
+                "unit": null
+            },
+            "hw:cpu_cores": {
+                "key": "numCpuCores",
+                "unit": null
+            },
+            "hw:cpu_threads": {
+                "key": "numCpuThreads",
+                "unit": null
+            }
+        }
+    },
+    "cpuPinning": {
+        "info": {
+            "hpa-feature": "cpuPinning",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:cpu_thread_policy": {
+                "key": "logicalCpuThreadPinningPolicy",
+                "unit": null
+            },
+            "hw:cpu_policy": {
+                "key": "logicalCpuPinningPolicy",
+                "unit": null
+            }
+        }
+    },
+    "numa": {
+        "info": {
+            "hpa-feature": "numa",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:numa_nodes": {
+                "key": "numaNodes",
+                "unit": null
+            },
+            "hw:numa_cpus": {
+                "key": "numaCpu",
+                "unit": null
+            },
+            "hw:numa_mem": {
+                "key": "numaMem",
+                "unit": "GB"
+            }
+        }
+    },
+    "hugePages": {
+        "info": {
+            "hpa-feature": "hugePages",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:mem_page_size": {
+                "key": "memoryPageSize",
+                "unit": null
+            }
+        }
+    },
+    "pciePassthrough": {
+        "info": {
+            "hpa-feature": "pciePassthrough",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "pci_count": {
+                "key": "pciCount",
+                "unit": null
+            },
+            "pci_vendor_id": {
+                "key": "pciVendorId",
+                "unit": null
+            },
+            "pci_device_id": {
+                "key": "pciDeviceId",
+                "unit": null
+            }
+        }
+    }
+}"""
+
 MOCK_GET_IMAGE_RESPONSE = {
     "images": [
         {
@@ -135,10 +280,13 @@ class TestFlavors(test_base.TestRequest):
         mock_response.json.return_value = return_value
         return mock_response
 
+    @mock.patch.object(Flavors, '_get_flavor_extra_specs')
     @mock.patch.object(VimDriverUtils, 'get_session')
     @mock.patch.object(VimDriverUtils, 'get_vim_info')
     def test_register_endpoint_successfully(
-            self, mock_get_vim_info, mock_get_session):
+            self, mock_get_vim_info, mock_get_session,
+            mock_get_extra_specs):
+        settings.AAI_SCHEMA_VERSION = "v13"
         restcall.req_to_aai = mock.Mock()
         restcall.req_to_aai.return_value = (0, {}, status.HTTP_200_OK)
         mock_get_vim_info.return_value = mock_info.MOCK_VIM_INFO
@@ -157,15 +305,20 @@ class TestFlavors(test_base.TestRequest):
                         MOCK_GET_HYPERVISOR_RESPONSE)
                 ]
             })
+        mock_extra_specs_response = mock.Mock(spec=test_base.MockResponse)
+        mock_extra_specs_response.status_code = status.HTTP_200_OK
+        mock_extra_specs_response.json.return_value = MOCK_GET_EXTRA_SPECS_RESPONSE
+        mock_get_extra_specs.return_value = mock_extra_specs_response
 
-        response = self.client.post((
-            "/api/%s/v0/windriver-hudson-dc_RegionOne/"
-            "registry" % test_base.MULTIVIM_VERSION),
-            TEST_REGISTER_ENDPOINT_REQUEST,
-            HTTP_X_AUTH_TOKEN=mock_info.MOCK_TOKEN_ID)
+        with mock.patch('__builtin__.open', mock.mock_open(read_data=MOCK_HPA_RESPONSE)) as mock_file:
+            response = self.client.post((
+                "/api/%s/v0/windriver-hudson-dc_RegionOne/"
+                "registry" % test_base.MULTIVIM_VERSION),
+                TEST_REGISTER_ENDPOINT_REQUEST,
+                HTTP_X_AUTH_TOKEN=mock_info.MOCK_TOKEN_ID)
 
-        self.assertEquals(status.HTTP_202_ACCEPTED,
-                          response.status_code)
+            self.assertEquals(status.HTTP_202_ACCEPTED,
+                              response.status_code)
 
     @mock.patch.object(VimDriverUtils, 'delete_vim_info')
     def test_unregister_endpoint_successfully(
diff --git a/newton/newton/registration/views/hpa.json b/newton/newton/registration/views/hpa.json
new file mode 100644
index 00000000..832d55af
--- /dev/null
+++ b/newton/newton/registration/views/hpa.json
@@ -0,0 +1,129 @@
+{
+    "basicCapabilities": {
+        "info": {
+            "hpa-feature": "basicCapabilities",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "vcpus": {
+                "key": "numVirtualCpu",
+                "unit": null
+            },
+            "ram": {
+                "key": "virtualMemSize",
+                "unit": "GB"
+            }
+        }
+    },
+    "localStorage": {
+        "info": {
+            "hpa-feature": "localStorage",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "disk": {
+                "key": "diskSize",
+                "unit": "GB"
+            },
+            "swap": {
+                "key": "swapMemSize",
+                "unit": "MB"
+            }
+        }
+    },
+    "cpuTopology": {
+        "info": {
+            "hpa-feature": "cpuTopology",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:cpu_sockets": {
+                "key": "numCpuSockets",
+                "unit": null
+            },
+            "hw:cpu_cores": {
+                "key": "numCpuCores",
+                "unit": null
+            },
+            "hw:cpu_threads": { 
+                "key": "numCpuThreads",
+                "unit": null
+            }
+        }
+    },
+    "cpuPinning": {
+        "info": {
+            "hpa-feature": "cpuPinning",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:cpu_thread_policy": {
+                "key": "logicalCpuThreadPinningPolicy",
+                "unit": null
+            },
+            "hw:cpu_policy": {
+                "key": "logicalCpuPinningPolicy",
+                "unit": null
+            }
+        }
+    },
+    "numa": {
+        "info": {
+            "hpa-feature": "numa",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:numa_nodes": {
+                "key": "numaNodes",
+                "unit": null
+            },
+            "hw:numa_cpus": {
+                "key": "numaCpu",
+                "unit": null
+            },
+            "hw:numa_mem": {
+                "key": "numaMem",
+                "unit": "GB"
+            }
+        }
+    },
+    "hugePages": {
+        "info": {
+            "hpa-feature": "hugePages",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "hw:mem_page_size": {
+                "key": "memoryPageSize",
+                "unit": null
+            }
+        }
+    },
+    "pciePassthrough": {
+        "info": {
+            "hpa-feature": "pciePassthrough",
+            "hpa-version": "v1",
+            "architecture": "generic"
+        },
+        "hpa-attributes": {
+            "pci_count": {
+                "key": "pciCount",
+                "unit": null
+            },
+            "pci_vendor_id": {
+                "key": "pciVendorId",
+                "unit": null
+            },
+            "pci_device_id": {
+                "key": "pciDeviceId",
+                "unit": null
+            }
+        }
+    }
+}
diff --git a/newton/newton/registration/views/registration.py b/newton/newton/registration/views/registration.py
index 69c85837..e9f1a357 100644
--- a/newton/newton/registration/views/registration.py
+++ b/newton/newton/registration/views/registration.py
@@ -13,10 +13,16 @@
 # limitations under the License.
 
 import logging
+import json
+import uuid
 
 from django.conf import settings
+from keystoneauth1.exceptions import HttpError
 
+from common.exceptions import VimDriverNewtonException
+from common.msapi import extsys
 from newton_base.registration import registration as newton_registration
+from newton_base.openoapi.flavor import Flavors
 
 logger = logging.getLogger(__name__)
 
@@ -28,3 +34,179 @@ class Registry(newton_registration.Registry):
         self.proxy_prefix = settings.MULTICLOUD_PREFIX
         self.aai_base_url = settings.AAI_BASE_URL
         self._logger = logger
+
+    def _discover_flavors(self, vimid="", session=None, viminfo=None):
+        try:
+            cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid)
+            for flavor in self._get_list_resources(
+                    "/flavors/detail", "compute", session, viminfo, vimid,
+                    "flavors"):
+                flavor_info = {
+                    'flavor-id': flavor['id'],
+                    'flavor-name': flavor['name'],
+                    'flavor-vcpus': flavor['vcpus'],
+                    'flavor-ram': flavor['ram'],
+                    'flavor-disk': flavor['disk'],
+                    'flavor-ephemeral': flavor['OS-FLV-EXT-DATA:ephemeral'],
+                    'flavor-swap': flavor['swap'],
+                    'flavor-is-public': flavor['os-flavor-access:is_public'],
+                    'flavor-disabled': flavor['OS-FLV-DISABLED:disabled'],
+                }
+                if flavor.get('link') and len(flavor['link']) > 0:
+                    flavor_info['flavor-selflink'] = flavor['link'][0]['href'] or 'http://0.0.0.0',
+                else:
+                    flavor_info['flavor-selflink'] = 'http://0.0.0.0',
+
+                if settings.AAI_SCHEMA_VERSION == "v13":
+                    extraResp = Flavors._get_flavor_extra_specs(session, flavor['id'])
+                    extraContent = extraResp.json()
+                    hpa_capabilities = self._get_hpa_capabilities(vimid, flavor,
+                                                                  extraContent["extra_specs"])
+                    flavor_info['hpa_capabilities'] = hpa_capabilities
+
+                self._update_resoure(
+                    cloud_owner, cloud_region_id, flavor['id'],
+                    flavor_info, "flavor")
+
+        except VimDriverNewtonException as e:
+            self._logger.error("VimDriverNewtonException: status:%s, response:%s" % (e.http_status, e.content))
+            return
+        except HttpError as e:
+            self._logger.error("HttpError: status:%s, response:%s" % (e.http_status, e.response.json()))
+            return
+        except Exception as e:
+            self._logger.error(traceback.format_exc())
+            return
+
+    def _get_hpa_capabilities(self, vimid, flavor, extra_specs):
+        """Convert flavor information to HPA capabilities for AAI"""
+        cloud_owner, cloud_region_id = extsys.decode_vim_id(vimid)
+
+        json_data = open('hpa.json').read()
+        hpa_dict = json.loads(json_data)
+
+        capabilities = []
+
+        # Basic Capabilities
+        if set(hpa_dict['basicCapabilities']['hpa-attributes']).intersection(flavor):
+            capability = hpa_dict['basicCapabilities']['info']
+            capability['hpa-capability-id'] = str(uuid.uuid4())
+            capability['hpa-feature-attributes'] = self._get_capability_attributes(
+                                                       flavor,
+                                                       hpa_dict['basicCapabilities']['hpa-attributes'])
+            capabilities.append(capability)
+
+        # Local Storage
+        if set(hpa_dict['localStorage']['hpa-attributes']).intersection(flavor):
+            capability = hpa_dict['localStorage']['info']
+            capability['hpa-capability-id'] = str(uuid.uuid4())
+            capability['hpa-feature-attributes'] = self._get_capability_attributes(
+                                                       flavor,
+                                                       hpa_dict['localStorage']['hpa-attributes'])
+            capabilities.append(capability)
+
+        # CPU Topology
+        if set(hpa_dict['cpuTopology']['hpa-attributes']).intersection(extra_specs):
+            capability = hpa_dict['cpuTopology']['info']
+            capability['hpa-capability-id'] = str(uuid.uuid4())
+            capability['hpa-features-attributes'] = self._get_capability_attributes(
+                                                        extra_specs,
+                                                        hpa_dict['cpuTopology']['hpa-attributes'])
+            capabilities.append(capability)
+
+        # CPU Pinning
+        if set(hpa_dict['cpuPinning']['hpa-attributes']).intersection(extra_specs):
+            capability = hpa_dict['cpuPinning']['info']
+            capability['hpa-capability-id'] = str(uuid.uuid4())
+            capability['hpa-features-attributes'] = self._get_capability_attributes(
+                                                        extra_specs,
+                                                        hpa_dict['cpuPinning']['hpa-attributes'])
+            capabilities.append(capability)
+
+        # Huge Pages
+        if set(hpa_dict['hugePages']['hpa-attributes']).intersection(extra_specs):
+            capability = hpa_dict['hugePages']['info']
+            if extra_specs['hw:mem_page_size'] not in ['small', 'large', 'any']:
+                unit = ''.join(i for i in extra_specs['hw:mem_page_size'] if not i.isdigit())
+                if unit == '':
+                    unit = 'KB'
+                hpa_dict['hugePages']['hpa-attributes']['hw:mem_page_size']['unit'] = unit
+            capability['hpa-capability-id'] = str(uuid.uuid4())
+            capability['hpa-features-attributes'] = self._get_capability_attributes(
+                                                        extra_specs,
+                                                        hpa_dict['hugePages']['hpa-attributes'])
+            capabilities.append(capability)
+
+        # NUMA
+        if "hw:numa_nodes" in extra_specs:
+            capability = hpa_dict['numa']['info']
+            capability['hpa-capability-id'] = str(uuid.uuid4())
+            # NUMA nodes are a special case and can't use the generic get attrs function
+            attributes = []
+            attributes.append({
+                'hpa-attribute-key': hpa_dict['numa']['hpa-attributes']['hw:numa_nodes']['key'],
+                'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(extra_specs['hw:numa_nodes'])
+            })
+            for spec in extra_specs:
+                if spec.startswith('hw:numa_cpus'):
+                    cpu_num = spec.split(":")[-1]
+                    attributes.append({
+                        'hpa-attribute-key': 'numaCpu-' + cpu_num,
+                        'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(extra_specs[spec])
+                    })
+                elif spec.startswith('hw:numa_mem'):
+                    mem_num = spec.split(":")[-1]
+                    attributes.append({
+                        'hpa-attribute-key': 'numaMem-' + mem_num,
+                        'hpa-attribute-value': '{{\"value\":\"{0}\",\"unit\":\"{1}\"}}'.format(extra_specs[spec],
+                                                                                               "GB")
+                    })
+            capability['hpa-features-attributes'] = attributes
+            capabilities.append(capability)
+
+        # PCIe Passthrough
+        pci_devices = [spec for spec in extra_specs if spec.startswith("pci_passthrough:alias")]
+        for device in pci_devices:
+            capability = hpa_dict['pciePassthrough']['info']
+            capability['hpa-capability-id'] = str(uuid.uuid4())
+            # device will be in the form pci_passthrough:alias=ALIAS:COUNT,
+            # ALIAS is expected to be in the form <NAME>-<VENDOR_ID>-<DEVICE_ID>
+            device_info = extra_specs[device].split(":")
+            count = device_info[-1]
+            vendor_id = device_info[0].split("-")[1]
+            device_id = device_info[0].split("-")[2]
+
+            attributes = [
+                {
+                    'hpa-attribute-key': 'pciCount',
+                    'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(count)
+                },
+                {
+                    'hpa-attribute-key': 'pciVendorId',
+                    'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(vendor_id)
+                },
+                {
+                    'hpa-attribute-key': 'pciDeviceId',
+                    'hpa-attribute-value': '{{\"value\":\"{0}\"}}'.format(device_id)
+                }
+            ]
+
+            capability['hpa-features-attributes'] = attributes
+            capabilities.append(capability)
+
+        return capabilities
+
+    def _get_capability_attributes(self, cloud_info, attributes):
+        result = []
+        for attr in attributes:
+            if attr in cloud_info:
+                attribute = {'hpa-attribute-key': attributes[attr]['key']}
+                if attributes[attr]['unit']:
+                    attribute['hpa-attribute-value'] = (
+                        '{{\"value\":\"{0}\",\"unit\":\"{1}\"}}').format(cloud_info[attr],
+                                                                         attributes[attr]['unit'])
+                else:
+                    attribute['hpa-attribute-value'] = '{{\"value\":\"{0}\"}}'.format(cloud_info[attr])
+
+                result.append(attribute)
+        return result