From 3437fc2b299bf73a525bbb30e9241af14c62bd57 Mon Sep 17 00:00:00 2001 From: Dileep Ranganathan Date: Thu, 15 Mar 2018 04:03:24 -0700 Subject: Data Service RPC for HPA Implemented RPC for get candidate list with hpa flavor mapping Added unit tests for get_candidates_with_hpa Issue-ID: OPTFRA-183 Change-Id: I9db74499ba964341368542b339163c3803e0924e Signed-off-by: Dileep Ranganathan --- conductor/conductor/data/service.py | 76 ++++++++- .../conductor/tests/unit/data/hpa_constraints.json | 171 +++++++++++++++++++++ .../conductor/tests/unit/data/test_service.py | 65 +++++++- 3 files changed, 305 insertions(+), 7 deletions(-) create mode 100644 conductor/conductor/tests/unit/data/hpa_constraints.json diff --git a/conductor/conductor/data/service.py b/conductor/conductor/data/service.py index acb4233..9617217 100644 --- a/conductor/conductor/data/service.py +++ b/conductor/conductor/data/service.py @@ -21,17 +21,17 @@ # import os import cotyledon -from oslo_config import cfg -from oslo_log import log -# from stevedore import driver - +from conductor import messaging # from conductor import __file__ as conductor_root from conductor.common.music import messaging as music_messaging +from conductor.common.utils import conductor_logging_util as log_util from conductor.data.plugins.inventory_provider import extensions as ip_ext from conductor.data.plugins.service_controller import extensions as sc_ext from conductor.i18n import _LE, _LI, _LW -from conductor import messaging -from conductor.common.utils import conductor_logging_util as log_util +from oslo_config import cfg +from oslo_log import log + +# from stevedore import driver # from conductor.solver.resource import region # from conductor.solver.resource import service @@ -427,6 +427,70 @@ class DataEndpoint(object): candidate_list, self.ip_ext_manager.names()[0])) return {'response': candidate_list, 'error': False} + def get_candidates_with_hpa(self, ctx, arg): + ''' + RPC for getting candidates flavor mapping for matching hpa + :param ctx: context + :param arg: contains input passed from client side for RPC call + :return: response candidate_list with matching label to flavor mapping + ''' + error = False + candidate_list = arg["candidate_list"] + label_name = arg["label_name"] + features = arg["features"] + discard_set = set() + for candidate in candidate_list: + # perform this check only for cloud candidates + if candidate["inventory_type"] != "cloud": + continue + + # Check if flavor mapping for current label_name already + # exists. This is an invalid condition. + if candidate.get("flavor_map") and candidate["flavor_map"].get( + label_name): + error = True + LOG.error(_LE("Flavor mapping for label name {} already" + "exists").format(label_name)) + return {'response': None, 'error': error} + + # RPC call to inventory provider for matching hpa capabilities + results = self.ip_ext_manager.map_method( + 'match_hpa', + candidate=candidate, + features=features + ) + + if results and len(results) > 0: + flavor_info = results[0] + else: + flavor_info = None + LOG.info( + _LW("No flavor mapping returned by " + "inventory provider: {} for candidate: {}").format( + self.ip_ext_manager.names()[0], + candidate.get("candidate_id"))) + if not flavor_info: + discard_set.add(candidate.get("candidate_id")) + else: + if not flavor_info.get("flavor-name"): + discard_set.add(candidate.get("candidate_id")) + else: + # Create flavor_map if not exist already + if not candidate.get("flavor_map"): + candidate["flavor_map"] = {} + # Create flavor mapping for label_name to flavor + flavor_name = flavor_info.get("flavor-name") + candidate["flavor_map"][label_name] = flavor_name + + # return candidates not in discard set + candidate_list[:] = [c for c in candidate_list + if c['candidate_id'] not in discard_set] + LOG.info(_LI( + "Candidates with matching hpa capabilities: {}, " + "inventory provider: {}").format(candidate_list, + self.ip_ext_manager.names()[0])) + return {'response': candidate_list, 'error': error} + def resolve_demands(self, ctx, arg): log_util.setLoggerFilter(LOG, ctx.get('keyspace'), ctx.get('plan_id')) diff --git a/conductor/conductor/tests/unit/data/hpa_constraints.json b/conductor/conductor/tests/unit/data/hpa_constraints.json new file mode 100644 index 0000000..3954cd5 --- /dev/null +++ b/conductor/conductor/tests/unit/data/hpa_constraints.json @@ -0,0 +1,171 @@ +{ + "conductor_solver": { + "constraints": [ + { + "hpa_constraint_vG": { + "demands": ["vG"], + "name": "hpa_constraint_vG", + "type": "hpa", + "properties": { + "evaluate": [ + { + "features": [ + { + "architecture": "generic", + "hpa-feature": "basicCapabilities", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numVirtualCpu", + "hpa-attribute-value": "4", + "operator": "=" + }, + { + "hpa-attribute-key": "virtualMemSize", + "hpa-attribute-value": "4", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "numa", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numaNodes", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-0", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-1", + "hpa-attribute-value": "4", + "operator": "=" + }, + { + "hpa-attribute-key": "numaMem-0", + "hpa-attribute-value": "2", + "operator": "=", + "unit": "GB" + }, + { + "hpa-attribute-key": "numaMem-1", + "hpa-attribute-value": "4", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "cpuPinning", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "logicalCpuThreadPinningPolicy", + "hpa-attribute-value": "prefer", + "operator": "=" + }, + { + "hpa-attribute-key": "logicalCpuPinningPolicy", + "hpa-attribute-value": "dedicated", + "operator": "=" + } + ], + "hpa-version": "v1" + } + ], + "label": "flavor_label_1" + }, + { + "features": [ + { + "architecture": "generic", + "hpa-feature": "basicCapabilities", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numVirtualCpu", + "hpa-attribute-value": "8", + "operator": "=" + }, + { + "hpa-attribute-key": "virtualMemSize", + "hpa-attribute-value": "16", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "numa", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "numaNodes", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-0", + "hpa-attribute-value": "2", + "operator": "=" + }, + { + "hpa-attribute-key": "numaCpu-1", + "hpa-attribute-value": "4", + "operator": "=" + }, + { + "hpa-attribute-key": "numaMem-0", + "hpa-attribute-value": "2", + "operator": "=", + "unit": "GB" + }, + { + "hpa-attribute-key": "numaMem-1", + "hpa-attribute-value": "4", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + }, + { + "architecture": "generic", + "hpa-feature": "memoryPageSize", + "hpa-feature-attributes": [ + { + "hpa-attribute-key": "memoryPageSize", + "hpa-attribute-value": "2", + "operator": "=", + "unit": "GB" + } + ], + "hpa-version": "v1" + } + ], + "label": "flavor_label_2" + } + ] + } + } + }, + { + "constraint_vgmux_customer": { + "type": "distance_to_location", + "demands": ["vGMuxInfra"], + "properties": { + "distance": "< 100 km", + "location": "customer_loc" + } + } + } + ] + } +} \ No newline at end of file diff --git a/conductor/conductor/tests/unit/data/test_service.py b/conductor/conductor/tests/unit/data/test_service.py index 89f1833..385b45d 100644 --- a/conductor/conductor/tests/unit/data/test_service.py +++ b/conductor/conductor/tests/unit/data/test_service.py @@ -16,6 +16,7 @@ # # ------------------------------------------------------------------------- # +import copy import json import unittest import uuid @@ -34,7 +35,6 @@ from oslo_config import cfg class TestDataEndpoint(unittest.TestCase): def setUp(self): - cfg.CONF.set_override('keyspace', 'conductor') ip_ext_manager = ( ip_ext.Manager(cfg.CONF, 'conductor.inventory_provider.plugin')) sc_ext_manager = ( @@ -218,6 +218,69 @@ class TestDataEndpoint(unittest.TestCase): self.assertEqual(expected_response, self.data_ep.resolve_demands(ctxt, req_json)) + @mock.patch.object(service.LOG, 'error') + @mock.patch.object(service.LOG, 'info') + @mock.patch.object(stevedore.ExtensionManager, 'names') + @mock.patch.object(stevedore.ExtensionManager, 'map_method') + def test_get_candidates_with_hpa(self, hpa_mock, ext_mock1, + info_mock, error_mock): + req_json_file = './conductor/tests/unit/data/candidate_list.json' + hpa_json_file = './conductor/tests/unit/data/hpa_constraints.json' + hpa_json = yaml.safe_load(open(hpa_json_file).read()) + req_json = yaml.safe_load(open(req_json_file).read()) + candidate_list = req_json['candidate_list'] + (constraint_id, constraint_info) = \ + hpa_json["conductor_solver"]["constraints"][0].items()[0] + hpa_constraint = constraint_info['properties'] + features = hpa_constraint['evaluate'][0]['features'] + label_name = hpa_constraint['evaluate'][0]['label'] + ext_mock1.return_value = ['aai'] + flavor_info = {"flavor-id": "vim-flavor-id1", + "flavor-name": "vim-flavor-name1"} + hpa_mock.return_value = [flavor_info] + self.maxDiff = None + args = generate_args(candidate_list, features, label_name) + hpa_candidate_list = copy.deepcopy(candidate_list) + hpa_candidate_list[1]['flavor_map'] = {} + hpa_candidate_list[1]['flavor_map'][label_name] = "vim-flavor-name1" + expected_response = {'response': hpa_candidate_list, 'error': False} + self.assertEqual(expected_response, + self.data_ep.get_candidates_with_hpa(None, args)) + + hpa_candidate_list2 = list() + hpa_candidate_list2.append(copy.deepcopy(candidate_list[0])) + args = generate_args(candidate_list, features, label_name) + hpa_mock.return_value = [] + expected_response = {'response': hpa_candidate_list2, 'error': False} + self.assertEqual(expected_response, + self.data_ep.get_candidates_with_hpa(None, args)) + + flavor_info = {} + hpa_mock.return_value = [flavor_info] + expected_response = {'response': hpa_candidate_list2, 'error': False} + self.assertEqual(expected_response, + self.data_ep.get_candidates_with_hpa(None, args)) + + flavor_info = {"flavor-id": "vim-flavor-id1", + "flavor-name": ""} + hpa_mock.return_value = [flavor_info] + expected_response = {'response': hpa_candidate_list2, 'error': False} + self.assertEqual(expected_response, + self.data_ep.get_candidates_with_hpa(None, args)) + + flavor_info = {"flavor-id": "vim-flavor-id1"} + hpa_mock.return_value = [flavor_info] + expected_response = {'response': hpa_candidate_list2, 'error': False} + self.assertEqual(expected_response, + self.data_ep.get_candidates_with_hpa(None, args)) + + +def generate_args(candidate_list, features, label_name): + arg_candidate_list = copy.deepcopy(candidate_list) + args = {"candidate_list": arg_candidate_list, + "features": features, + "label_name": label_name} + return args def ip_ext_sideeffect(*args, **kwargs): req_json_file = './conductor/tests/unit/data/constraints.json' -- cgit 1.2.3-korg