From fc3ead31e631f69fabf0baaa20c10bf955ce374b Mon Sep 17 00:00:00 2001 From: Lukasz Rajewski Date: Mon, 8 Apr 2019 15:05:22 +0200 Subject: Traffic Distributtion support added * New local polcies for vFW TD use case * Fixed encoding for conductor_request template and parameters section modified to accept all requestParameters * Conductor request can have many attributes in the 'attributes' section - all that are defined in the vnf policy file * Conductor request can have many request parameters in the 'requestParameters' section. The parameters come from QueryPolicies. Before list of suppoted parameters was hardcoded * Optional 'unique' parameter added to the placementDemand section. It is already supported by conductor for all inventory types * Improved debug logs for local policies * Unit tests added for expanded request format Change-Id: I41f219c366a3a77881c7096e64a6272edbada23b Issue-ID: OPTFRA-443 Signed-off-by: Lukasz Rajewski --- osdf/adapters/local_data/local_policies.py | 3 +- osdf/adapters/policy/interface.py | 4 ++ osdf/models/api/placementRequest.py | 3 +- .../placementopt/conductor/api_builder.py | 32 +++++++-- .../placementopt/conductor/translation.py | 27 ++++++++ osdf/templates/conductor_interface.json | 80 +++++++++++----------- 6 files changed, 99 insertions(+), 50 deletions(-) (limited to 'osdf') diff --git a/osdf/adapters/local_data/local_policies.py b/osdf/adapters/local_data/local_policies.py index 6e49388..dc6837a 100644 --- a/osdf/adapters/local_data/local_policies.py +++ b/osdf/adapters/local_data/local_policies.py @@ -19,7 +19,7 @@ import json import os import re - +from osdf.logging.osdf_logging import debug_log def get_local_policies(local_policy_folder, local_policy_list, policy_id_list=None): """ @@ -32,6 +32,7 @@ def get_local_policies(local_policy_folder, local_policy_list, policy_id_list=No :param policy_id_list: list of policies to get (if unspecified or None, get all) :return: get policies """ + debug_log.debug("Policy folder: {}, local_list {}, policy id list {}".format(local_policy_folder, local_policy_list, policy_id_list)) policies = [] if policy_id_list: for policy_id in policy_id_list: diff --git a/osdf/adapters/policy/interface.py b/osdf/adapters/policy/interface.py index 95bfacc..a7839c6 100644 --- a/osdf/adapters/policy/interface.py +++ b/osdf/adapters/policy/interface.py @@ -157,10 +157,12 @@ def local_policies_location(req_json, osdf_config, service_type): if lp.get('global_disabled'): return None # short-circuit to disable all local policies if lp.get('local_{}_policies_enabled'.format(service_type)): + debug_log.debug('Loading local policies for service type: {}'.format(service_type)) if service_type == "scheduling": return lp.get('{}_policy_dir'.format(service_type)), lp.get('{}_policy_files'.format(service_type)) else: service_name = req_json['serviceInfo']['serviceName'] # TODO: data_mapping.get_service_type(model_name) + debug_log.debug('Loading local policies for service name: {}'.format(service_name)) return lp.get('{}_policy_dir_{}'.format(service_type, service_name.lower())), \ lp.get('{}_policy_files_{}'.format(service_type, service_name.lower())) return None @@ -178,6 +180,8 @@ def get_policies(request_json, service_type): local_info = local_policies_location(request_json, osdf_config, service_type) if local_info: # tuple containing location and list of files + if local_info[0] is None or local_info[1] is None: + raise ValueError("Error fetching local policy info") to_filter = None if osdf_config.core['policy_info'][service_type]['policy_fetch'] == "by_name": to_filter = request_json[service_type + "Info"]['policyId'] diff --git a/osdf/models/api/placementRequest.py b/osdf/models/api/placementRequest.py index 55f0a98..7d6bde4 100644 --- a/osdf/models/api/placementRequest.py +++ b/osdf/models/api/placementRequest.py @@ -17,7 +17,7 @@ # from .common import OSDFModel -from schematics.types import BaseType, StringType, URLType, IntType +from schematics.types import BaseType, StringType, URLType, IntType, BooleanType from schematics.types.compound import ModelType, ListType, DictType @@ -71,6 +71,7 @@ class PlacementDemand(OSDFModel): resourceModuleName = StringType(required=True) serviceResourceId = StringType(required=True) tenantId = StringType() + unique = BooleanType() # to be implemented on the policy level resourceModelInfo = ModelType(ModelMetaData, required=True) existingCandidates = ListType(ModelType(Candidates)) excludedCandidates = ListType(ModelType(Candidates)) diff --git a/osdf/optimizers/placementopt/conductor/api_builder.py b/osdf/optimizers/placementopt/conductor/api_builder.py index 187f9f5..08a7460 100644 --- a/osdf/optimizers/placementopt/conductor/api_builder.py +++ b/osdf/optimizers/placementopt/conductor/api_builder.py @@ -25,6 +25,29 @@ from osdf.adapters.policy.utils import group_policies_gen from osdf.utils.programming_utils import list_flatten +def _build_parameters(group_policies, request_json): + """ + Function prepares parameters section for has request + :param group_policies: filtered policies + :param request_json: parameter data received from a client + :return: + """ + initial_params = tr.get_opt_query_data(request_json, group_policies['request_param_query']) + params = dict() + params.update({"REQUIRED_MEM": initial_params.pop("requiredMemory", "")}) + params.update({"REQUIRED_DISK": initial_params.pop("requiredDisk", "")}) + params.update({"customer_lat": initial_params.pop("customerLatitude", 0.0)}) + params.update({"customer_long": initial_params.pop("customerLongitude", 0.0)}) + params.update({"service_name": request_json['serviceInfo']['serviceName']}) + params.update({"service_id": request_json['serviceInfo']['serviceInstanceId']}) + + for key, val in initial_params.items(): + if val and val != "": + params.update({key: val}) + + return params + + def conductor_api_builder(request_json, flat_policies: list, local_config, template="osdf/templates/conductor_interface.json"): """Build an OSDF southbound API call for HAS-Conductor/Placement optimization @@ -54,7 +77,7 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, reservation_policy_list = tr.gen_reservation_policy(demand_vnf_name_list, gp['instance_reservation']) capacity_policy_list = tr.gen_capacity_policy(demand_vnf_name_list, gp['vim_fit']) hpa_policy_list = tr.gen_hpa_policy(demand_vnf_name_list, gp['hpa']) - req_params_dict = tr.get_opt_query_data(request_json, gp['request_param_query']) + req_params_dict = _build_parameters(gp, request_json) conductor_policies = [attribute_policy_list, distance_to_location_policy_list, inventory_policy_list, resource_instance_policy_list, resource_region_policy_list, zone_policy_list, reservation_policy_list, capacity_policy_list, hpa_policy_list] @@ -70,12 +93,7 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, name=req_info['requestId'], timeout=req_info['timeout'], limit=req_info['numSolutions'], - service_type=request_json['serviceInfo']['serviceName'], - service_id=request_json['serviceInfo']['serviceInstanceId'], - latitude=req_params_dict.get("customerLatitude", 0.0), - longitude=req_params_dict.get("customerLongitude", 0.0), - required_disk=req_params_dict.get("requiredDisk", ""), - required_mem=req_params_dict.get("requiredMemory", ""), + request_params=req_params_dict, json=json) json_payload = json.dumps(json.loads(rendered_req)) # need this because template's JSON is ugly! return json_payload diff --git a/osdf/optimizers/placementopt/conductor/translation.py b/osdf/optimizers/placementopt/conductor/translation.py index 93b80bf..d14f3e1 100644 --- a/osdf/optimizers/placementopt/conductor/translation.py +++ b/osdf/optimizers/placementopt/conductor/translation.py @@ -18,6 +18,7 @@ import copy import json import yaml +import re from osdf.utils.data_conversion import text_to_symbol from osdf.utils.programming_utils import dot_notation @@ -227,6 +228,8 @@ def get_demand_properties(demand, policies): inventory_type=policy_property['inventoryType'], service_type=demand['serviceResourceId'], service_resource_id=demand['serviceResourceId']) + + prop.update({'unique': demand['unique']} if demand.get('unique') else {}) prop['attributes'] = dict() prop['attributes'].update({'global-customer-id': policy_property['customerId']} if policy_property['customerId'] else {}) @@ -236,11 +239,35 @@ def get_demand_properties(demand, policies): if demand['resourceModelInfo']['modelVersionId'] else {}) prop['attributes'].update({'equipment-role': policy_property['equipmentRole']} if policy_property['equipmentRole'] else {}) + + if policy_property.get('attributes'): + for attr_key, attr_val in policy_property['attributes'].items(): + update_converted_attribute(attr_key, attr_val, prop) + prop.update(get_candidates_demands(demand)) demand_properties.append(prop) return demand_properties +def update_converted_attribute(attr_key, attr_val, properties): + """ + Updates dictonary of attributes with one specified in the arguments. + Automatically translates key namr from camelCase to hyphens + :param attr_key: key of the attribute + :param attr_val: value of the attribute + :param properties: dictionary with attributes to update + :return: + """ + if attr_val: + remapping = policy_config_mapping['attributes'] + if remapping.get(attr_key): + key_value = remapping.get(attr_key) + else: + key_value = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', attr_key) + key_value = re.sub('([a-z0-9])([A-Z])', r'\1-\2', key_value).lower() + properties['attributes'].update({key_value: attr_val}) + + def gen_demands(req_json, vnf_policies): """Generate list of demands based on request and VNF policies :param req_json: Request object from the client (e.g. MSO) diff --git a/osdf/templates/conductor_interface.json b/osdf/templates/conductor_interface.json index 7377c48..0b8e6a1 100755 --- a/osdf/templates/conductor_interface.json +++ b/osdf/templates/conductor_interface.json @@ -1,41 +1,39 @@ -{ - "name": "{{ name }}", - "files": {}, - "timeout": {{ timeout }}, - "limit": {{ limit }}, - "template": { - "homing_template_version": "2017-10-10", - "parameters": { - "service_name": "{{ service_type }}", - "service_id": "{{ service_id }}", - "customer_lat": {{ latitude }}, - "customer_long": {{ longitude }}, - "REQUIRED_DISK": "{{ required_disk }}", - "REQUIRED_MEM": "{{ required_mem }}" - }, - "locations": { - "customer_loc": { - "latitude": { "get_param": "customer_lat" }, - "longitude": { "get_param": "customer_long" } - } - }, - "demands": {{ json.dumps(demand_list) }}, - {% set comma_main = joiner(",") %} - "constraints": { - {% set comma=joiner(",") %} - {% for elem in policy_groups %} {{ comma() }} - {% for key, value in elem.items() %} - "{{key}}": {{ json.dumps(value) }} - {% endfor %} - {% endfor %} - }, - "optimization": { - {% set comma=joiner(",") %} - {% for elem in optimization_policies %} {{ comma() }} - {% for key, value in elem.items() %} - "{{key}}": {{ json.dumps(value) }} - {% endfor %} - {% endfor %} - } - } -} +{ + "name": "{{ name }}", + "files": {}, + "timeout": {{ timeout }}, + "limit": {{ limit }}, + "template": { + "homing_template_version": "2017-10-10", + "parameters": { + {% set comma=joiner(",") %} + {% for key, value in request_params.items() %} {{ comma() }} + "{{key}}": {{ json.dumps(value) }} + {% endfor %} + }, + "locations": { + "customer_loc": { + "latitude": { "get_param": "customer_lat" }, + "longitude": { "get_param": "customer_long" } + } + }, + "demands": {{ json.dumps(demand_list) }}, + {% set comma_main = joiner(",") %} + "constraints": { + {% set comma=joiner(",") %} + {% for elem in policy_groups %} {{ comma() }} + {% for key, value in elem.items() %} + "{{key}}": {{ json.dumps(value) }} + {% endfor %} + {% endfor %} + }, + "optimization": { + {% set comma=joiner(",") %} + {% for elem in optimization_policies %} {{ comma() }} + {% for key, value in elem.items() %} + "{{key}}": {{ json.dumps(value) }} + {% endfor %} + {% endfor %} + } + } +} -- cgit 1.2.3-korg