diff options
Diffstat (limited to 'conductor')
5 files changed, 752 insertions, 2 deletions
diff --git a/conductor/conductor/data/plugins/inventory_provider/aai.py b/conductor/conductor/data/plugins/inventory_provider/aai.py index c40e15c..41f55c5 100644 --- a/conductor/conductor/data/plugins/inventory_provider/aai.py +++ b/conductor/conductor/data/plugins/inventory_provider/aai.py @@ -24,6 +24,7 @@ import uuid from conductor.common import rest from conductor.data.plugins import constants from conductor.data.plugins.inventory_provider import base +from conductor.data.plugins.inventory_provider import hpa_utils from conductor.i18n import _LE, _LI from oslo_config import cfg from oslo_log import log @@ -1300,3 +1301,10 @@ class AAI(base.InventoryProviderBase): " {}".format(inventory_type)) return resolved_demands + + def match_hpa(self, candidate, features): + """Match HPA features requirement with the candidate flavors """ + hpa_provider = hpa_utils.HpaMatchProvider(candidate, features) + flavor_map = hpa_provider.match_flavor() + return flavor_map + diff --git a/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py b/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py index 6ed622c..41e217c 100644 --- a/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py +++ b/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py @@ -22,8 +22,10 @@ Hardware Platform Awareness (HPA) constraint plugin''' # python imports +import yaml +import operator -# Conductor imports +from conductor.i18n import _LE, _LI # Third-party library imports from oslo_log import log @@ -46,3 +48,222 @@ def match_all_operator(big_list, small_list): small_set = set(small_list) return small_set.issubset(big_set) + + +class HpaMatchProvider(object): + + def __init__(self, candidate, req_cap_list): + self.flavors_list = candidate['flavors']['flavor'] + self.req_cap_list = req_cap_list + + # Find the flavor which has all the required capabilities + def match_flavor(self): + # Keys to find capability match + hpa_keys = ['hpa-feature', 'architecture', 'hpa-version'] + req_filter_list = [] + for capability in CapabilityDataParser.get_item(self.req_cap_list, + None): + if capability.item['mandatory'] == 'True': + hpa_list = {k: capability.item[k] \ + for k in hpa_keys if k in capability.item} + req_filter_list.append(hpa_list) + max_score = -1 + flavor_map = None + for flavor in self.flavors_list: + flavor_filter_list = [] + try: + flavor_cap_list = flavor['hpa-capabilities'] + except KeyError: + LOG.info(_LI("invalid JSON ")) + return None + for capability in CapabilityDataParser.get_item(flavor_cap_list, + 'hpa-capability'): + hpa_list = {k: capability.item[k] \ + for k in hpa_keys if k in capability.item} + flavor_filter_list.append(hpa_list) + # if flavor has the matching capability compare attributes + if self._is_cap_supported(flavor_filter_list, req_filter_list): + match_found, score = self._compare_feature_attributes(flavor_cap_list) + if match_found: + LOG.info(_LI("Matching Flavor found '{}' for request - {}"). + format(flavor['flavor-name'], self.req_cap_list)) + if score > max_score: + max_score = score + flavor_map = {"flavor-id": flavor['flavor-id'], + "flavor-name": flavor['flavor-name']} + return flavor_map + + + def _is_cap_supported(self, flavor, cap): + try: + for elem in cap: + flavor.remove(elem) + except ValueError: + return False + # Found all capabilities in Flavor + return True + + # Convert to bytes value using unit + def _get_normalized_value(self, unit, value): + + if not value.isdigit(): + return value + value = int(value) + if unit == 'KB': + value = value * 1024 + elif unit == 'MB': + value = value * 1024 * 1024 + elif unit == 'GB': + value = value * 1024 * 1024 * 1024 + return str(value) + + def _get_req_attribute(self, req_attr): + try: + c_op = req_attr['operator'] + c_value = req_attr['hpa-attribute-value'] + c_unit = None + if 'unit' in req_attr: + c_unit = req_attr['unit'] + except KeyError: + LOG.info(_LI("invalid JSON ")) + return None + + if c_unit: + c_value = self._get_normalized_value(c_unit, c_value) + return c_value, c_op + + def _get_flavor_attribute(self, flavor_attr): + try: + attrib_value = yaml.load(flavor_attr['hpa-attribute-value']) + except: + return None + + f_unit = None + f_value = None + for key, value in attrib_value.iteritems(): + if key == 'value': + f_value = value + elif key == 'unit': + f_unit = value + if f_unit: + f_value = self._get_normalized_value(f_unit, f_value) + return f_value + + def _get_operator(self, req_op): + + operator_list = ['=', '<', '>', '<=', '>=', 'ALL'] + + if req_op not in operator_list: + return None + + if req_op == ">": + op = operator.gt + elif req_op == ">=": + op = operator.ge + elif req_op == "<": + op = operator.lt + elif req_op == "<=": + op = operator.le + elif req_op == "=": + op = operator.eq + elif req_op == 'ALL': + op = match_all_operator + + return op + + + def _compare_attribute(self, flavor_attr, req_attr): + + req_value, req_op = self._get_req_attribute(req_attr) + flavor_value = self._get_flavor_attribute(flavor_attr) + + if req_value is None or flavor_value is None: + return False + + # Compare operators only valid for Integers + if req_op in ['<', '>', '<=', '>=']: + if not req_value.isdigit() or not flavor_value.isdigit(): + return False + + op = self._get_operator(req_op) + if not op: + return False + + if req_op == 'ALL': + # All is valid only for lists + if isinstance(req_value, list) and isinstance(flavor_value, list): + return op(flavor_value, req_value) + + # if values are string compare them as strings + if req_op == '=': + if not req_value.isdigit() or not flavor_value.isdigit(): + return op(req_value, flavor_value) + + # Only integers left to compare + if req_op in ['<', '>', '<=', '>=', '=']: + return op(int(flavor_value), int(req_value)) + + return False + + # for the feature get the capabilty feature attribute list + def _get_flavor_cfa_list(self, feature, flavor_cap_list): + for capability in CapabilityDataParser.get_item(flavor_cap_list, + 'hpa-capability'): + flavor_feature, feature_attributes = capability.get_fields() + # One feature will match this condition as we have pre-filtered + if feature == flavor_feature: + return feature_attributes + + # flavor has all the required capabilties + # For each required capability find capability in flavor + # and compare each attribute + def _compare_feature_attributes(self, flavor_cap_list): + score = 0 + for capability in CapabilityDataParser.get_item(self.req_cap_list, None): + hpa_feature, req_cfa_list = capability.get_fields() + flavor_cfa_list = self._get_flavor_cfa_list(hpa_feature, flavor_cap_list) + if flavor_cfa_list is not None: + for req_feature_attr in req_cfa_list: + req_attr_key = req_feature_attr['hpa-attribute-key'] + # filter to get the attribute being compared + flavor_feature_attr = \ + filter(lambda ele: ele['hpa-attribute-key'] == \ + req_attr_key, flavor_cfa_list) + if not flavor_feature_attr: + return False, 0 + if not self._compare_attribute(flavor_feature_attr[0], + req_feature_attr): + return False, 0 + if flavor_cfa_list is not None and capability.item['mandatory'] == 'False': + score = score + int(capability.item['score']) + return True, score + + +class CapabilityDataParser(object): + """Helper class to parse data""" + + def __init__(self, item): + self.item = item + + @classmethod + def get_item(cls, payload, key): + try: + if key is None: + features = payload + else: + features = (payload[key]) + + for f in features: + yield cls(f) + except KeyError: + LOG.info(_LI("invalid JSON ")) + + def get_fields(self): + return (self.get_feature(), + self.get_feature_attributes()) + + def get_feature_attributes(self): + return self.item.get('hpa-feature-attributes') + + def get_feature(self): + return self.item.get('hpa-feature') diff --git a/conductor/conductor/tests/unit/data/plugins/inventory_provider/hpa_flavors.json b/conductor/conductor/tests/unit/data/plugins/inventory_provider/hpa_flavors.json new file mode 100644 index 0000000..db5ea54 --- /dev/null +++ b/conductor/conductor/tests/unit/data/plugins/inventory_provider/hpa_flavors.json @@ -0,0 +1,297 @@ +{ + "flavor": [ + { + "flavor-id": "9cf8220b-4d96-4c30-a426-2e9382f3fff2", + "flavor-name": "flavor-numa-cpu-topology-instruction-set", + "flavor-vcpus": 64, + "flavor-ram": 65536, + "flavor-disk": 1048576, + "flavor-ephemeral": 128, + "flavor-swap": "0", + "flavor-is-public": false, + "flavor-selflink": "pXtX", + "flavor-disabled": false, + "hpa-capabilities": { + "hpa-capability": [ + { + "hpa-capability-id": "13ec6d4d-7fee-48d8-9e4a-c598feb101ed", + "hpa-feature": "basicCapabilities", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943845409", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numVirtualCpu", + "hpa-attribute-value": "{value:4}", + "resource-version": "1520943845416" + }, + { + "hpa-attribute-key": "virtualMemSize", + "hpa-attribute-value": "{value:4, unit:\"GB\" }", + "resource-version": "1520943845427" + } + ] + }, + { + "hpa-capability-id": "01a4bfe1-1993-4fda-bd1c-ef333b4f76a9", + "hpa-feature": "instructionSetExtensions", + "hpa-version": "v1", + "architecture": "Intel64", + "resource-version": "1520943846236", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "instructionSetExtensions", + "hpa-attribute-value": "{value:{['AAA', 'BBB', 'CCC', 'DDD']}}", + "resource-version": "1520943846241" + } + ] + }, + { + "hpa-capability-id": "167ad6a2-7d9c-4bf2-9a1b-30e5311b8c66", + "hpa-feature": "numa", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943846158", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numaCpu-0", + "hpa-attribute-value": "{value:2}", + "resource-version": "1520943846178" + }, + { + "hpa-attribute-key": "numaMem-0", + "hpa-attribute-value": "{value:2, unit:\"GB\" }", + "resource-version": "1520943846204" + }, + { + "hpa-attribute-key": "numaCpu-1", + "hpa-attribute-value": "{value:4}", + "resource-version": "1520943846191" + }, + { + "hpa-attribute-key": "numaMem-1", + "hpa-attribute-value": "{value:4, unit:\"GB\" }", + "resource-version": "1520943846217" + }, + { + "hpa-attribute-key": "numaNodes", + "hpa-attribute-value": "{value:2}", + "resource-version": "1520943846163" + } + ] + }, + { + "hpa-capability-id": "8fa22e64-41b4-471f-96ad-6c4708635e4c", + "hpa-feature": "cpuTopology", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943845443", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numCpuCores", + "hpa-attribute-value": "{value:8}", + "resource-version": "1520943845456" + }, + { + "hpa-attribute-key": "numCpuSockets", + "hpa-attribute-value": "{value:6}", + "resource-version": "1520943845447" + }, + { + "hpa-attribute-key": "numCpuThreads", + "hpa-attribute-value": "{value:8}", + "resource-version": "1520943846129" + } + ] + } + ] + }, + "resource-version": "1520943845399" + }, + { + "flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43", + "flavor-name": "flavor-cpu-ovsdpdk-instruction-set", + "flavor-vcpus": 32, + "flavor-ram": 131072, + "flavor-disk": 2097152, + "flavor-ephemeral": 128, + "flavor-swap": "0", + "flavor-is-public": false, + "flavor-selflink": "pXtX", + "flavor-disabled": false, + "hpa-capabilities": { + "hpa-capability": [ + { + "hpa-capability-id": "8d36a8fe-bfee-446a-bbcb-881ee66c8f78", + "hpa-feature": "ovsDpdk", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943846328", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "dataProcessingAccelerationLibrary", + "hpa-attribute-value": "{\"value\":\"v18.02\"}", + "resource-version": "1520943846346" + } + ] + }, + { + "hpa-capability-id": "4d04f4d8-e257-4442-8417-19a525e56096", + "hpa-feature": "instructionSetExtensions", + "hpa-version": "v1", + "architecture": "Intel64", + "resource-version": "1520943846362", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "instructionSetExtensions", + "hpa-attribute-value": "{\"value\" : [\"A11\", \"B22\"] }", + "resource-version": "1520943846365" + } + ] + }, + { + "hpa-capability-id": "4565615b-1077-4bb5-a340-c5be48db2aaa", + "hpa-feature": "basicCapabilities", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943846269", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "virtualMemSize", + "hpa-attribute-value": "{\"value\":\"16\", \"unit\":\"GB\" }", + "resource-version": "1520943846282" + }, + { + "hpa-attribute-key": "numVirtualCpu", + "hpa-attribute-value": "{\"value\":\"8\"}", + "resource-version": "1520943846272" + } + ] + }, + { + "hpa-capability-id": "8fa22e64-41b4-471f-96ad-6c4708635e4c", + "hpa-feature": "cpuTopology", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943845443", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numCpuCores", + "hpa-attribute-value": "{\"value\":\"4\"}", + "resource-version": "1520943845456" + }, + { + "hpa-attribute-key": "numCpuSockets", + "hpa-attribute-value": "{\"value\": \"4\"}", + "resource-version": "1520943845447" + }, + { + "hpa-attribute-key": "numCpuThreads", + "hpa-attribute-value": "{\"value\": \"8\"}", + "resource-version": "1520943846129" + } + ] + } + + ] + }, + "resource-version": "1520943846264" + }, + { + "flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43", + "flavor-name": "flavor-cpu-pinning-ovsdpdk-instruction-set", + "flavor-vcpus": 32, + "flavor-ram": 131072, + "flavor-disk": 2097152, + "flavor-ephemeral": 128, + "flavor-swap": "0", + "flavor-is-public": false, + "flavor-selflink": "pXtX", + "flavor-disabled": false, + "hpa-capabilities": { + "hpa-capability": [ + { + "hpa-capability-id": "8d36a8fe-bfee-446a-bbcb-881ee66c8f78", + "hpa-feature": "ovsDpdk", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943846328", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "dataProcessingAccelerationLibrary", + "hpa-attribute-value": "{\"value\":\"v18.02\"}", + "resource-version": "1520943846346" + } + ] + }, + { + "hpa-capability-id": "c140c945-1532-4908-86c9-d7f71416f1dd", + "hpa-feature": "cpuPinning", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943846297", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "logicalCpuPinningPolicy", + "hpa-attribute-value": "{\"value\":\"dedicated\"}", + "resource-version": "1520943846312" + }, + { + "hpa-attribute-key": "logicalCpuThreadPinningPolicy", + "hpa-attribute-value": "{\"value\":\"prefer\"}", + "resource-version": "1520943846301" + } + ] + }, + { + "hpa-capability-id": "4565615b-1077-4bb5-a340-c5be48db2aaa", + "hpa-feature": "basicCapabilities", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943846269", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "virtualMemSize", + "hpa-attribute-value": "{\"value\":\"16\", \"unit\":\"GB\" }", + "resource-version": "1520943846282" + }, + { + "hpa-attribute-key": "numVirtualCpu", + "hpa-attribute-value": "{\"value\":\"8\"}", + "resource-version": "1520943846272" + } + ] + }, + { + "hpa-capability-id": "8fa22e64-41b4-471f-96ad-6c4708635e4c", + "hpa-feature": "cpuTopology", + "hpa-version": "v1", + "architecture": "generic", + "resource-version": "1520943845443", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numCpuCores", + "hpa-attribute-value": "{\"value\":\"4\"}", + "resource-version": "1520943845456" + }, + { + "hpa-attribute-key": "numCpuSockets", + "hpa-attribute-value": "{\"value\": \"4\"}", + "resource-version": "1520943845447" + }, + { + "hpa-attribute-key": "numCpuThreads", + "hpa-attribute-value": "{\"value\": \"8\"}", + "resource-version": "1520943846129" + } + ] + } + + ] + }, + "resource-version": "1520943846264" + } + + ] +} + diff --git a/conductor/conductor/tests/unit/data/plugins/inventory_provider/hpa_req_features.json b/conductor/conductor/tests/unit/data/plugins/inventory_provider/hpa_req_features.json new file mode 100644 index 0000000..4a5d37a --- /dev/null +++ b/conductor/conductor/tests/unit/data/plugins/inventory_provider/hpa_req_features.json @@ -0,0 +1,200 @@ +[ + [ + { + "hpa-feature":"basicCapabilities", + "hpa-version":"v1", + "architecture":"generic", + "mandatory":"True", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"numVirtualCpu", + "hpa-attribute-value":"8", + "operator":"=" + }, + { + "hpa-attribute-key":"virtualMemSize", + "hpa-attribute-value":"16384", + "operator":"=", + "unit":"MB" + } + ] + }, + { + "hpa-feature":"cpuPinning", + "hpa-version":"v1", + "architecture":"generic", + "mandatory":"True", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"logicalCpuThreadPinningPolicy", + "hpa-attribute-value":"prefer", + "operator":"=" + }, + { + "hpa-attribute-key":"logicalCpuPinningPolicy", + "hpa-attribute-value":"dedicated", + "operator":"=" + } + ] + }, + { + "hpa-feature":"cpuTopology", + "hpa-version":"v1", + "mandatory":"True", + "architecture":"generic", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"numCpuSockets", + "hpa-attribute-value":"2", + "operator":">=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuSockets", + "hpa-attribute-value":"4", + "operator":"<=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuCores", + "hpa-attribute-value":"2", + "operator":">=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuCores", + "hpa-attribute-value":"4", + "operator":"<=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuThreads", + "hpa-attribute-value":"4", + "operator":">=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuThreads", + "hpa-attribute-value":"8", + "operator":"<=", + "unit":"" + } + ] + } + ], + [ + { + "hpa-feature":"basicCapabilities", + "hpa-version":"v1", + "architecture":"generic", + "mandatory":"True", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"numVirtualCpu", + "hpa-attribute-value":"8", + "operator":"=" + }, + { + "hpa-attribute-key":"virtualMemSize", + "hpa-attribute-value":"16384", + "operator":"=", + "unit":"MB" + } + ] + }, + { + "hpa-feature":"ovsDpdk", + "hpa-version":"v1", + "architecture":"generic", + "mandatory":"False", + "score":"5", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"dataProcessingAccelerationLibrary", + "hpa-attribute-value":"v18.02", + "operator":"=" + } + ] + }, + { + "hpa-feature":"cpuPinning", + "hpa-version":"v1", + "architecture":"generic", + "mandatory":"False", + "score":"1", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"logicalCpuThreadPinningPolicy", + "hpa-attribute-value":"prefer", + "operator":"=" + }, + { + "hpa-attribute-key":"logicalCpuPinningPolicy", + "hpa-attribute-value":"dedicated", + "operator":"=" + } + ] + }, + { + "hpa-feature":"instructionSetExtensions", + "hpa-version":"v1", + "architecture":"Intel64", + "mandatory":"False", + "score":"5", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"instructionSetExtensions", + "hpa-attribute-value":[ + "A11", + "B22" + ], + "operator":"ALL" + } + ] + }, + { + "hpa-feature":"cpuTopology", + "hpa-version":"v1", + "mandatory":"True", + "architecture":"generic", + "hpa-feature-attributes":[ + { + "hpa-attribute-key":"numCpuSockets", + "hpa-attribute-value":"2", + "operator":">=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuSockets", + "hpa-attribute-value":"4", + "operator":"<=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuCores", + "hpa-attribute-value":"2", + "operator":">=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuCores", + "hpa-attribute-value":"4", + "operator":"<=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuThreads", + "hpa-attribute-value":"4", + "operator":">=", + "unit":"" + }, + { + "hpa-attribute-key":"numCpuThreads", + "hpa-attribute-value":"8", + "operator":"<=", + "unit":"" + } + ] + } + ] +]
\ No newline at end of file diff --git a/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py b/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py index b148579..e12a114 100644 --- a/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py +++ b/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py @@ -297,4 +297,28 @@ class TestAAI(unittest.TestCase): flavors_info = self.aai_ep._get_flavors("mock-cloud-owner", "mock-cloud-region-id") - self.assertEqual(2, len(flavors_info['flavor']))
\ No newline at end of file + self.assertEqual(2, len(flavors_info['flavor'])) + + def test_match_hpa(self): + flavor_json_file = \ + './conductor/tests/unit/data/plugins/inventory_provider/hpa_flavors.json' + flavor_json = json.loads(open(flavor_json_file).read()) + feature_json_file = \ + './conductor/tests/unit/data/plugins/inventory_provider/hpa_req_features.json' + feature_json = json.loads(open(feature_json_file).read()) + candidate_json_file = './conductor/tests/unit/data/candidate_list.json' + candidate_json = json.loads(open(candidate_json_file).read()) + candidate_json['candidate_list'][1]['flavors'] = flavor_json + + flavor_map = {"flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43", + "flavor-name": "flavor-cpu-pinning-ovsdpdk-instruction-set"} + self.assertEqual(flavor_map, + self.aai_ep.match_hpa(candidate_json['candidate_list'][1], + feature_json[0])) + + flavor_map = {"flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43", + "flavor-name": "flavor-cpu-ovsdpdk-instruction-set" } + self.assertEqual(flavor_map, + self.aai_ep.match_hpa(candidate_json['candidate_list'][1], + feature_json[1])) + |