From f1e95cd95c05be145c3fde9f08404724937edf49 Mon Sep 17 00:00:00 2001 From: Ankitkumar Patel Date: Tue, 27 Mar 2018 11:54:53 -0400 Subject: Updates to address new HPA policies Issue-ID: OPTFRA-100 Change-Id: Iaff7b50dc5394546cc46591bef9b054cd6cde792 Signed-off-by: Ankitkumar Patel --- osdf/adapters/policy/utils.py | 38 ++++++++++++-- .../placementopt/conductor/api_builder.py | 43 ++++++++-------- .../placementopt/conductor/translation.py | 24 ++++----- osdf/templates/conductor_interface.json | 58 ++++------------------ 4 files changed, 76 insertions(+), 87 deletions(-) (limited to 'osdf') diff --git a/osdf/adapters/policy/utils.py b/osdf/adapters/policy/utils.py index 9acfa2a..95d03af 100644 --- a/osdf/adapters/policy/utils.py +++ b/osdf/adapters/policy/utils.py @@ -19,7 +19,7 @@ import copy import json from collections import defaultdict - +import itertools from osdf.utils.programming_utils import dot_notation, list_flatten @@ -33,13 +33,13 @@ def group_policies(flat_policies): """ filtered_policies = defaultdict(list) policy_name = [] - policies = [x for x in flat_policies if x['content'].get('policy_type')] # drop ones without 'policy_type' + policies = [x for x in flat_policies if x['content'].get('policyType')] # drop ones without 'policy_type' policy_types = set([x['content'].get('policyType') for x in policies]) aggregated_policies = dict((x, defaultdict(list)) for x in policy_types) for policy in policies: policy_type = policy['content'].get('policyType') - for resource in policy['content'].get('resourceInstanceType', []): + for resource in policy['content'].get('resources', []): aggregated_policies[policy_type][resource].append(policy) for policy_type in aggregated_policies: @@ -54,6 +54,38 @@ def group_policies(flat_policies): return filtered_policies +def group_policies_gen(flat_policies, config): + """Filter policies using the following steps: + 1. Apply prioritization among the policies that are sharing the same policy type and resource type + 2. Remove redundant policies that may applicable across different types of resource + 3. Filter policies based on type and return + :param flat_policies: list of flat policies + :return: Filtered policies + """ + filtered_policies = defaultdict(list) + policy_name = [] + policies = [x for x in flat_policies if x['content'].get('policyType')] # drop ones without 'policy_type' + priority = config.get('policy_info', {}).get('prioritization_attributes', {}) + aggregated_policies = dict() + for plc in policies: + attrs = [dot_notation(plc, dot_path) for key in priority.keys() for dot_path in priority[key]] + attrs_list = [x if isinstance(x, list) else [x] for x in attrs] + attributes = [list_flatten(x) if isinstance(x, list) else x for x in attrs_list] + for y in itertools.product(*attributes): + aggregated_policies.setdefault(y, []) + aggregated_policies[y].append(plc) + + for key in aggregated_policies.keys(): + aggregated_policies[key].sort(key=lambda x: x['priority'], reverse=True) + prioritized_policy = aggregated_policies[key][0] + if prioritized_policy['policyName'] not in policy_name: + # TODO: Check logic here... should policy appear only once across all groups? + filtered_policies[prioritized_policy['content']['policyType']].append(prioritized_policy) + policy_name.append(prioritized_policy['policyName']) + + return filtered_policies + + def policy_name_as_regex(policy_name): """Get the correct policy name as a regex (e.g. OOF_HAS_vCPE.cloudAttributePolicy ends up in policy as OOF_HAS_vCPE.Config_MS_cloudAttributePolicy.1.xml diff --git a/osdf/optimizers/placementopt/conductor/api_builder.py b/osdf/optimizers/placementopt/conductor/api_builder.py index bfc7f19..55044d0 100644 --- a/osdf/optimizers/placementopt/conductor/api_builder.py +++ b/osdf/optimizers/placementopt/conductor/api_builder.py @@ -21,12 +21,11 @@ import json from jinja2 import Template import osdf.optimizers.placementopt.conductor.translation as tr -from osdf.adapters.policy.utils import group_policies +from osdf.adapters.policy.utils import group_policies_gen from osdf.utils.programming_utils import list_flatten -def conductor_api_builder(request_json, flat_policies: list, local_config, prov_status, - template="templates/conductor_interface.json"): +def conductor_api_builder(request_json, flat_policies: list, local_config, template="templates/conductor_interface.json"): """Build an OSDF southbound API call for HAS-Conductor/Placement optimization :param request_json: parameter data received from a client :param flat_policies: policy data received from the policy platform (flat policies) @@ -36,7 +35,7 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_ :return: json to be sent to Conductor/placement optimization """ templ = Template(open(template).read()) - gp = group_policies(flat_policies) + gp = group_policies_gen(flat_policies, local_config) demand_vnf_name_list = [] for placementDemand in request_json['placementInfo']['placementDemands']: @@ -60,31 +59,29 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_ reservation_policies = [x for x in reservation_policy_list if len(x) > 0] reservation_groups = list_flatten(reservation_policies) req_info = request_json['requestInfo'] - model_name = request_json['serviceInfo']['serviceName'] - service_type = model_name + request_type = req_info.get('requestType', None) + service_type = request_json['serviceInfo']['serviceName'] service_info = local_config.get('service_info', {}).get(service_type, {}) order_info = {} if 'orderInfo' in request_json["placementInfo"]: order_info = json.loads(request_json["placementInfo"]["orderInfo"]) - request_type = req_info.get('requestType', None) + subs_com_site_id = "" if 'subscriberInfo' in request_json['placementInfo']: subs_com_site_id = request_json['placementInfo']['subscriberInfo'].get('subscriberCommonSiteId', "") - rendered_req = None - if service_type == 'vCPE': - rendered_req = templ.render( - requestType=request_type, - chosenComplex=subs_com_site_id, - demand_list=demand_list, - policy_groups=policy_groups, - optimization_policies=optimization_policy_list, - name=req_info['requestId'], - timeout=req_info['timeout'], - limit=req_info['numSolutions'], - serviceType=service_type, - serviceInstance=request_json['serviceInfo']['serviceInstanceId'], - provStatus=prov_status, - chosenRegion=order_info.get('requestParameters', {}).get('lcpCloudRegionId'), - json=json) + rendered_req = templ.render( + requestType=request_type, + chosenComplex=subs_com_site_id, + demand_list=demand_list, + policy_groups=policy_groups, + optimization_policies=optimization_policy_list, + name=req_info['requestId'], + timeout=req_info['timeout'], + limit=req_info['numSolutions'], + service_type=request_json['serviceInfo']['serviceName'], + service_id=request_json['serviceInfo']['serviceInstanceId'], + provStatus="", + chosenRegion=order_info.get('requestParameters', {}).get('lcpCloudRegionId'), + json=json) json_payload = json.dumps(json.loads(rendered_req)) # need this because template's JSON is ugly! return json_payload diff --git a/osdf/optimizers/placementopt/conductor/translation.py b/osdf/optimizers/placementopt/conductor/translation.py index 9111c81..91706b7 100644 --- a/osdf/optimizers/placementopt/conductor/translation.py +++ b/osdf/optimizers/placementopt/conductor/translation.py @@ -37,8 +37,8 @@ def gen_optimization_policy(vnf_list, optimization_policy): for attr in content['objectiveParameter']['parameterAttributes']: parameter = attr['parameter'] if attr['parameter'] == "cloud_version" else attr['parameter']+"_between" - for res in attr['resource']: - vnf = get_matching_vnf(res, vnf_list) + vnfs = get_matching_vnfs(attr['resources'], vnf_list) + for vnf in vnfs: value = [vnf] if attr['parameter'] == "cloud_version" else [attr['customerLocationInfo'], vnf] parameter_list.append({ attr['operator']: [attr['weight'], {parameter: value}] @@ -66,7 +66,7 @@ def get_matching_vnfs(resources, vnf_list, match_type="intersection"): :return: List of matching VNFs """ if match_type == "all": # don't bother with any comparisons - return resources + return resources if set(resources) <= set(vnf_list) else None common_vnfs = set(vnf_list) & set(resources) if match_type == "intersection": # specifically requested intersection return list(common_vnfs) @@ -87,8 +87,8 @@ def gen_policy_instance(vnf_list, resource_policy, match_type="intersection", rt related_policies = [] for policy in resource_policy: pc = policy['content'] - demands = get_matching_vnfs(pc['resourceInstanceType'], vnf_list, match_type=match_type) - resource = {pc['identity']: {'type': pc['type'], 'demands': demands}} + demands = get_matching_vnfs(pc['resources'], vnf_list, match_type=match_type) + resource = {pc['identity']: {'type': pc['policyType'], 'demands': demands}} if rtype: resource[pc['identity']]['properties'] = {'controller': pc[rtype]['controller'], @@ -127,10 +127,10 @@ def gen_distance_to_location_policy(vnf_list, distance_to_location_policy): """Get policies governing distance-to-location for VNFs in order to populate the Conductor API call""" cur_policies, related_policies = gen_policy_instance(vnf_list, distance_to_location_policy, rtype=None) for p_new, p_main in zip(cur_policies, related_policies): # add additional fields to each policy - properties = p_main['content']['distanceToLocationProperty'] - pcp_d = properties['distanceCondition'] + properties = p_main['content']['distanceProperties'] + pcp_d = properties['distance'] p_new[p_main['content']['identity']]['properties'] = { - 'distance': text_to_symbol[pcp_d['operator']] + " " + pcp_d['value'].lower(), + 'distance': pcp_d['operator'] + " " + pcp_d['value'].lower() + " " + pcp_d['unit'].lower(), 'location': properties['locationInfo'] } return cur_policies @@ -150,9 +150,9 @@ def gen_attribute_policy(vnf_list, attribute_policy): def gen_zone_policy(vnf_list, zone_policy): """Get zone policies in order to populate the Conductor API call""" - cur_policies, related_policies = gen_policy_instance(vnf_list, zone_policy, rtype=None) + cur_policies, related_policies = gen_policy_instance(vnf_list, zone_policy, match_type="all", rtype=None) for p_new, p_main in zip(cur_policies, related_policies): # add additional fields to each policy - pmz = p_main['content']['zoneProperty'] + pmz = p_main['content']['affinityProperty'] p_new[p_main['content']['identity']]['properties'] = {'category': pmz['category'], 'qualifier': pmz['qualifier']} return cur_policies @@ -179,9 +179,9 @@ def get_candidates_demands(demand): def get_policy_properties(demand, policies): """Get policy_properties for cases where there is a match with the demand""" for policy in policies: - if not set(policy['content'].get('resourceInstanceType', [])) & set(demand['resourceModuleName']): + if demand['resourceModuleName'] not in set(policy['content'].get('resources', [])): continue # no match for this policy - for policy_property in policy['content']['property']: + for policy_property in policy['content']['vnfProperties']: yield policy_property diff --git a/osdf/templates/conductor_interface.json b/osdf/templates/conductor_interface.json index 2b48647..64f9c4f 100755 --- a/osdf/templates/conductor_interface.json +++ b/osdf/templates/conductor_interface.json @@ -4,52 +4,20 @@ "timeout": {{ timeout }}, "limit": {{ limit }}, "template": { - "conductor_template_version": "2018-02-01", + "homing_template_version": "2017-10-10", "parameters": { - "REQUEST_TYPE": "{{ requestType }}", - "CHOSEN_REGION": "{{ chosenRegion }}", - "LATITUDE": "{{ latitude }}", - "LONGITUDE": "{{ longitude }}", - {% if serviceType == 'DHV' %} - "E2EVPNKEY": "{{ e2eVpnKey }}", - "UCPEHOST": "{{ ucpeHostName }}", - "EFFECTIVE_BANDWIDTH": "{{ effectiveBandwidth }}", - "WAN_PORT1_UP": "{{ ipsec_bw_up }}", - "WAN_PORT1_DOWN": "{{ ipsec_bw_down }}", - "WAN_PORT2_UP": "{{ ipsec2_bw_up }}", - "WAN_PORT2_DOWN": "{{ ipsec2_bw_down }}", - {% endif %} - {% if serviceType != 'DHV' %} - "GW_TENANT_ID": "{{ gwTenantId }}", - "PORTAL_TENANT_ID": "{{ portalTenantId }}", - {% endif %} - "CHOSEN_COMPLEX": "{{ chosenComplex }}", - {% if serviceType == 'ADIOD' or serviceType == 'VPE' %} - "BANDWIDTH": "{{ bandwidth }}", - "UNIT": "{{ bandwidth_unit }}", - {% endif %} - "SERVICE_INST": "{{ serviceInstance }}", - "PROV_STATUS": {{ json.dumps(provStatus) }} + "service_name": "{{ service_name }}", + "service_id": "{{ service_id }}", + "customer_lat": "{{ latitude }}", + "customer_long": "{{ longitude }}", + "REQUIRED_MEM": "{{ memory }}", + "REQUIRED_DISK": "{{ storage }}" }, "locations": { - {% if serviceType == 'DHV' %} "customer_loc": { - "host_name": { "get_param": "UCPEHOST" } + "latitude": { "get_param": "customer_lat" }, + "longitude": { "get_param": "customer_long" } } - {% elif serviceType == 'ADIOD' %} - "customer_pref_location": { - "clli_code": { "get_param": "CHOSEN_COMPLEX" } - } - {% elif serviceType == 'NETBOND' %} - "peering_point": { - "latitude": { "get_param": "LATITUDE" }, - "longitude": { "get_param": "LONGITUDE" } - } - {% else %} - "customer_pref_loc": { - "clli_code": { "get_param": "CHOSEN_COMPLEX" } - } - {% endif %} }, "demands": {{ json.dumps(demand_list) }}, {% set comma_main = joiner(",") %} @@ -61,14 +29,6 @@ {% endfor %} {% endfor %} }, - "reservation": { - {% set comma=joiner(",") %} - {% for elem in reservation_groups %} {{ comma() }} - {% for key, value in elem.items() %} - "{{key}}": {{ json.dumps(value) }} - {% endfor %} - {% endfor %} - }, "optimization": { {% set comma=joiner(",") %} {% for elem in optimization_policies %} {{ comma() }} -- cgit 1.2.3-korg