From 6ec674c56f4965606a4f865ac327c30d3ce4a53e Mon Sep 17 00:00:00 2001 From: Dileep Ranganathan Date: Wed, 22 Aug 2018 09:18:49 -0700 Subject: HPA Score Objective function support Implemented HPA Score objective function for cross cloud-region best candidate selection. Added Unit tests for HPA Score multi objective function Added Unit tests to check if there is a corresponding HPA policy/constraint Change-Id: I0787080657c0b7deb3ffcb7560859e3e5c928b77 Issue-ID: OPTFRA-313 Signed-off-by: Dileep Ranganathan --- conductor/conductor/controller/translator.py | 22 ++- .../data/plugins/inventory_provider/hpa_utils.py | 3 +- conductor/conductor/data/service.py | 5 + .../solver/request/functions/hpa_score.py | 29 ++++ conductor/conductor/solver/request/objective.py | 5 + conductor/conductor/solver/request/parser.py | 4 + .../tests/unit/controller/hpa_constraints.json | 191 +++++++++++++++++++++ .../tests/unit/controller/test_translator.py | 56 ++++++ .../data/plugins/inventory_provider/test_aai.py | 6 +- .../conductor/tests/unit/data/test_service.py | 4 +- 10 files changed, 319 insertions(+), 6 deletions(-) create mode 100755 conductor/conductor/solver/request/functions/hpa_score.py create mode 100644 conductor/conductor/tests/unit/controller/hpa_constraints.json diff --git a/conductor/conductor/controller/translator.py b/conductor/conductor/controller/translator.py index 7cd90f4..fb591e0 100644 --- a/conductor/conductor/controller/translator.py +++ b/conductor/conductor/controller/translator.py @@ -649,7 +649,7 @@ class Translator(object): "No value specified for property '{}' in " "constraint named '{}'".format( req_prop, name)) - # For HPA constraints + # For HPA constraints if constraint_type == 'hpa': self.validate_hpa_constraints(req_prop, value) @@ -794,6 +794,13 @@ class Translator(object): elif product_op.keys() == ['aic_version']: function = 'aic_version' args = product_op.get('aic_version') + elif product_op.keys() == ['hpa_score']: + function = 'hpa_score' + args = product_op.get('hpa_score') + if not self.is_hpa_policy_exists(args): + raise TranslatorException( + "HPA Score Optimization must include a " + "HPA Policy constraint ") elif product_op.keys() == ['sum']: nested = True nested_operands = product_op.get('sum') @@ -844,6 +851,18 @@ class Translator(object): ) return parsed + def is_hpa_policy_exists(self, demand_list): + # Check if a HPA constraint exist for the demands in the demand list. + constraints_copy = copy.deepcopy(self._constraints) + for demand in demand_list: + for name, constraint in constraints_copy.items(): + constraint_type = constraint.get('type') + if constraint_type == 'hpa': + hpa_demands = constraint.get('demands') + if demand in hpa_demands: + return True + return False + def parse_reservations(self, reservations): demands = self._demands if type(reservations) is not dict: @@ -880,7 +899,6 @@ class Translator(object): "request_type": request_type, "locations": self.parse_locations(self._locations), "demands": self.parse_demands(self._demands), - "objective": self.parse_optimization(self._optmization), "constraints": self.parse_constraints(self._constraints), "objective": self.parse_optimization(self._optmization), "reservations": self.parse_reservations(self._reservations), diff --git a/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py b/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py index 24f901b..648775a 100644 --- a/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py +++ b/conductor/conductor/data/plugins/inventory_provider/hpa_utils.py @@ -91,7 +91,8 @@ class HpaMatchProvider(object): if score > max_score: max_score = score flavor_map = {"flavor-id": flavor['flavor-id'], - "flavor-name": flavor['flavor-name']} + "flavor-name": flavor['flavor-name'], + "score": max_score} return flavor_map diff --git a/conductor/conductor/data/service.py b/conductor/conductor/data/service.py index e9d597b..0af7bb7 100644 --- a/conductor/conductor/data/service.py +++ b/conductor/conductor/data/service.py @@ -486,6 +486,11 @@ class DataEndpoint(object): # Create flavor mapping for label_name to flavor flavor_name = flavor_info.get("flavor-name") candidate["flavor_map"][label_name] = flavor_name + # If hpa_score is not defined then initialize value 0 + # hpa_score = sum of scores of each vnfc hpa requirement score + if not candidate.get("hpa_score"): + candidate["hpa_score"] = 0 + candidate["hpa_score"] += flavor_info.get("score") # return candidates not in discard set candidate_list[:] = [c for c in candidate_list diff --git a/conductor/conductor/solver/request/functions/hpa_score.py b/conductor/conductor/solver/request/functions/hpa_score.py new file mode 100755 index 0000000..b09e4a7 --- /dev/null +++ b/conductor/conductor/solver/request/functions/hpa_score.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# +# ------------------------------------------------------------------------- +# Copyright (c) 2018 Intel Corporation Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +'''Objective function for hpa_score + Hardware Platform Awareness (HPA)''' + + +class HPAScore(object): + + def __init__(self, _type): + self.func_type = _type + self.score = 0 diff --git a/conductor/conductor/solver/request/objective.py b/conductor/conductor/solver/request/objective.py index 0559056..f255330 100755 --- a/conductor/conductor/solver/request/objective.py +++ b/conductor/conductor/solver/request/objective.py @@ -109,6 +109,11 @@ class Operand(object): for demand_name, candidate_info in _decision_path.decisions.items(): value += float(candidate_info['cost']) + elif self.function.func_type == "hpa_score": + for demand_name, candidate_info in _decision_path.decisions.items(): + hpa_score = float(candidate_info.get('hpa_score', 0)) + value += hpa_score + if self.operation == "product": value *= self.weight diff --git a/conductor/conductor/solver/request/parser.py b/conductor/conductor/solver/request/parser.py index 0def215..da031cc 100755 --- a/conductor/conductor/solver/request/parser.py +++ b/conductor/conductor/solver/request/parser.py @@ -41,6 +41,7 @@ from conductor.solver.request import objective from conductor.solver.request.functions import aic_version from conductor.solver.request.functions import cost from conductor.solver.request.functions import distance_between +from conductor.solver.request.functions import hpa_score from oslo_log import log # from conductor.solver.request.functions import distance_between @@ -263,6 +264,9 @@ class Parser(object): func = cost.Cost("cost") func.loc = operand_data["function_param"] operand.function = func + elif operand_data["function"] == "hpa_score": + func = hpa_score.HPAScore("hpa_score") + operand.function = func self.objective.operand_list.append(operand) diff --git a/conductor/conductor/tests/unit/controller/hpa_constraints.json b/conductor/conductor/tests/unit/controller/hpa_constraints.json new file mode 100644 index 0000000..5a86ede --- /dev/null +++ b/conductor/conductor/tests/unit/controller/hpa_constraints.json @@ -0,0 +1,191 @@ +{ + "HAS_Template": { + "constraints": { + "hpa_constraint_vG": { + "demands": [ + "vG" + ], + "name": "hpa_constraint_vG", + "type": "hpa", + "properties": { + "evaluate": [ + { + "flavorProperties": [ + { + "architecture": "generic", + "hpa-feature": "basicCapabilities", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numVirtualCpu", + "hpa-attribute-value": "4", + "operator": "=" + }, + { + "hpa-attribute-key": "virtualMemSize", + "hpa-attribute-value": "4", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "numa", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numaNodes", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-0", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-1", + "hpa-attribute-value": "4", + "operator": "=" + }, + { + "hpa-attribute-key": "numaMem-0", + "hpa-attribute-value": "2", + "operator": "=", + "unit": "GB" + }, + { + "hpa-attribute-key": "numaMem-1", + "hpa-attribute-value": "4", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "cpuPinning", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "logicalCpuThreadPinningPolicy", + "hpa-attribute-value": "prefer", + "operator": "=" + }, + { + "hpa-attribute-key": "logicalCpuPinningPolicy", + "hpa-attribute-value": "dedicated", + "operator": "=" + } + ], + "hpa-version": "v1" + } + ], + "flavorLabel": "flavor_label_1" + }, + { + "flavorProperties": [ + { + "architecture": "generic", + "hpa-feature": "basicCapabilities", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numVirtualCpu", + "hpa-attribute-value": "8", + "operator": "=" + }, + { + "hpa-attribute-key": "virtualMemSize", + "hpa-attribute-value": "16", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "numa", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numaNodes", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-0", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-1", + "hpa-attribute-value": "4", + "operator": "=" + }, + { + "hpa-attribute-key": "numaMem-0", + "hpa-attribute-value": "2", + "operator": "=", + "unit": "GB" + }, + { + "hpa-attribute-key": "numaMem-1", + "hpa-attribute-value": "4", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "memoryPageSize", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "memoryPageSize", + "hpa-attribute-value": "2", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + } + ], + "flavorLabel": "flavor_label_2" + } + ] + } + }, + "constraint_vgmux_customer": { + "type": "distance_to_location", + "demands": [ + "vGMuxInfra" + ], + "properties": { + "distance": "< 100 km", + "location": "customer_loc" + } + }, + "check_cloud_capacity": { + "type": "vim_fit", + "demands": [ + "vG" + ], + "properties": { + "controller": "multicloud", + "request": { + "vCPU": 10, + "Memory": { + "quantity": "10", + "unit": "GB" + }, + "Storage": { + "quantity": "100", + "unit": "GB" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/conductor/conductor/tests/unit/controller/test_translator.py b/conductor/conductor/tests/unit/controller/test_translator.py index ba0e3ec..e6575a9 100644 --- a/conductor/conductor/tests/unit/controller/test_translator.py +++ b/conductor/conductor/tests/unit/controller/test_translator.py @@ -572,6 +572,62 @@ class TestNoExceptionTranslator(unittest.TestCase): self.Translator.parse_optimization( opt), expected_parse) + @patch('conductor.controller.translator.Translator.create_components') + def test_parse_optimization_multi_objective(self, mock_create): + hpa_json_file = './conductor/tests/unit/controller/hpa_constraints.json' + hpa_json = yaml.safe_load(open(hpa_json_file).read()) + expected_parse = {'goal': 'min', + 'operands': [{'function': 'distance_between', + 'function_param': ['customer_loc', + 'vGMuxInfra'], + 'operation': 'product', + 'weight': 2.0}, + {'function': 'distance_between', + 'function_param': ['customer_loc', + 'vG'], + 'operation': 'product', + 'weight': 4.0}, + {'function': 'hpa_score', + 'function_param': ['vG'], + 'operation': 'product', + 'weight': 8.0}, + ], + 'operation': 'sum' + } + + opt = {'minimize': { + 'sum': [{'product': [2.0, {'distance_between': ['customer_loc', 'vGMuxInfra']}]}, + {'product': [4.0, {'distance_between': ['customer_loc', 'vG']}]}, + {'product': [8.0, {'hpa_score': ['vG']}]} + ]}} + self.Translator._demands = {'vG': '', + 'vGMuxInfra': '', + 'customer_loc': ''} + self.Translator._locations = {'vG': '', + 'vGMuxInfra': '', + 'customer_loc': ''} + self.Translator._constraints = hpa_json["HAS_Template"]["constraints"] + self.maxDiff = None + self.assertEquals( + self.Translator.parse_optimization( + opt), expected_parse) + + # No HPA Policy test + non_hpa_dict = dict(hpa_json["HAS_Template"]["constraints"]) + non_hpa_dict.pop("hpa_constraint_vG") + self.Translator._constraints = non_hpa_dict + self.maxDiff = None + self.assertRaises(TranslatorException, + self.Translator.parse_optimization, opt) + + # HPA Policy Exists but not for the demand in objective function + hpa_wrong_demand_dict = dict(hpa_json["HAS_Template"]["constraints"]) + hpa_wrong_demand_dict["hpa_constraint_vG"]["demands"] = ["vGMuxInfra"] + self.Translator._constraints = hpa_wrong_demand_dict + self.maxDiff = None + self.assertRaises(TranslatorException, + self.Translator.parse_optimization, opt) + @patch('conductor.controller.translator.Translator.create_components') def test_parse_reservation(self, mock_create): expected_resv = {'counter': 0, 'demands': { diff --git a/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py b/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py index e12a114..5713d04 100644 --- a/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py +++ b/conductor/conductor/tests/unit/data/plugins/inventory_provider/test_aai.py @@ -311,13 +311,15 @@ class TestAAI(unittest.TestCase): candidate_json['candidate_list'][1]['flavors'] = flavor_json flavor_map = {"flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43", - "flavor-name": "flavor-cpu-pinning-ovsdpdk-instruction-set"} + "flavor-name": "flavor-cpu-pinning-ovsdpdk-instruction-set", + "score": 0} self.assertEqual(flavor_map, self.aai_ep.match_hpa(candidate_json['candidate_list'][1], feature_json[0])) flavor_map = {"flavor-id": "f5aa2b2e-3206-41b6-80d5-cf041b098c43", - "flavor-name": "flavor-cpu-ovsdpdk-instruction-set" } + "flavor-name": "flavor-cpu-ovsdpdk-instruction-set", + "score": 10} self.assertEqual(flavor_map, self.aai_ep.match_hpa(candidate_json['candidate_list'][1], feature_json[1])) diff --git a/conductor/conductor/tests/unit/data/test_service.py b/conductor/conductor/tests/unit/data/test_service.py index 01c2ab3..8c74097 100644 --- a/conductor/conductor/tests/unit/data/test_service.py +++ b/conductor/conductor/tests/unit/data/test_service.py @@ -241,13 +241,15 @@ class TestDataEndpoint(unittest.TestCase): label_name = hpa_constraint['evaluate'][0]['flavorLabel'] ext_mock1.return_value = ['aai'] flavor_info = {"flavor-id": "vim-flavor-id1", - "flavor-name": "vim-flavor-name1"} + "flavor-name": "vim-flavor-name1", + "score": 0} hpa_mock.return_value = [flavor_info] self.maxDiff = None args = generate_args(candidate_list, flavorProperties, label_name) hpa_candidate_list = copy.deepcopy(candidate_list) hpa_candidate_list[1]['flavor_map'] = {} hpa_candidate_list[1]['flavor_map'][label_name] = "vim-flavor-name1" + hpa_candidate_list[1]['hpa_score'] = 0 expected_response = {'response': hpa_candidate_list, 'error': False} self.assertEqual(expected_response, self.data_ep.get_candidates_with_hpa(None, args)) -- cgit 1.2.3-korg