Updates to address new HPA policies

Issue-ID: OPTFRA-100
Change-Id: Iaff7b50dc5394546cc46591bef9b054cd6cde792
Signed-off-by: Ankitkumar Patel <ankit@research.att.com>

4 files changed, 76 insertions, 87 deletions

diff --git a/osdf/adapters/policy/utils.py b/osdf/adapters/policy/utils.py
index 9acfa2a..95d03af 100644
--- a/osdf/adapters/policy/utils.py
+++ b/osdf/adapters/policy/utils.py
@@ -19,7 +19,7 @@ import copy
 import json
 
 from collections import defaultdict
-
+import itertools
 from osdf.utils.programming_utils import dot_notation, list_flatten
 
 
@@ -33,13 +33,13 @@ def group_policies(flat_policies):
     """
     filtered_policies = defaultdict(list)
     policy_name = []
-    policies = [x for x in flat_policies if x['content'].get('policy_type')]  # drop ones without 'policy_type'
+    policies = [x for x in flat_policies if x['content'].get('policyType')]  # drop ones without 'policy_type'
     policy_types = set([x['content'].get('policyType') for x in policies])
     aggregated_policies = dict((x, defaultdict(list)) for x in policy_types)
 
     for policy in policies:
         policy_type = policy['content'].get('policyType')
-        for resource in policy['content'].get('resourceInstanceType', []):
+        for resource in policy['content'].get('resources', []):
             aggregated_policies[policy_type][resource].append(policy)
 
     for policy_type in aggregated_policies:
@@ -54,6 +54,38 @@ def group_policies(flat_policies):
     return filtered_policies
 
 
+def group_policies_gen(flat_policies, config):
+    """Filter policies using the following steps:
+    1. Apply prioritization among the policies that are sharing the same policy type and resource type
+    2. Remove redundant policies that may applicable across different types of resource
+    3. Filter policies based on type and return
+    :param flat_policies: list of flat policies
+    :return: Filtered policies
+    """
+    filtered_policies = defaultdict(list)
+    policy_name = []
+    policies = [x for x in flat_policies if x['content'].get('policyType')]  # drop ones without 'policy_type'
+    priority = config.get('policy_info', {}).get('prioritization_attributes', {})
+    aggregated_policies = dict()
+    for plc in policies:
+        attrs = [dot_notation(plc, dot_path) for key in priority.keys() for dot_path in priority[key]]
+        attrs_list  = [x if isinstance(x, list) else [x] for x in attrs]
+        attributes = [list_flatten(x) if isinstance(x, list) else x for x in attrs_list]
+        for y in itertools.product(*attributes):
+            aggregated_policies.setdefault(y, [])
+            aggregated_policies[y].append(plc)
+
+    for key in aggregated_policies.keys():
+        aggregated_policies[key].sort(key=lambda x: x['priority'], reverse=True)
+        prioritized_policy = aggregated_policies[key][0]
+        if prioritized_policy['policyName'] not in policy_name:
+            # TODO: Check logic here... should policy appear only once across all groups?
+            filtered_policies[prioritized_policy['content']['policyType']].append(prioritized_policy)
+            policy_name.append(prioritized_policy['policyName'])
+
+    return filtered_policies
+
+
 def policy_name_as_regex(policy_name):
     """Get the correct policy name as a regex
     (e.g. OOF_HAS_vCPE.cloudAttributePolicy ends up in policy as OOF_HAS_vCPE.Config_MS_cloudAttributePolicy.1.xml
diff --git a/osdf/optimizers/placementopt/conductor/api_builder.py b/osdf/optimizers/placementopt/conductor/api_builder.py
index bfc7f19..55044d0 100644
--- a/osdf/optimizers/placementopt/conductor/api_builder.py
+++ b/osdf/optimizers/placementopt/conductor/api_builder.py
@@ -21,12 +21,11 @@ import json
 from jinja2 import Template
 
 import osdf.optimizers.placementopt.conductor.translation as tr
-from osdf.adapters.policy.utils import group_policies
+from osdf.adapters.policy.utils import group_policies_gen
 from osdf.utils.programming_utils import list_flatten
 
 
-def conductor_api_builder(request_json, flat_policies: list, local_config, prov_status,
-                          template="templates/conductor_interface.json"):
+def conductor_api_builder(request_json, flat_policies: list, local_config, template="templates/conductor_interface.json"):
     """Build an OSDF southbound API call for HAS-Conductor/Placement optimization
     :param request_json: parameter data received from a client
     :param flat_policies: policy data received from the policy platform (flat policies)
@@ -36,7 +35,7 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_
     :return: json to be sent to Conductor/placement optimization
     """
     templ = Template(open(template).read())
-    gp = group_policies(flat_policies)
+    gp = group_policies_gen(flat_policies, local_config)
     demand_vnf_name_list = []
 
     for placementDemand in request_json['placementInfo']['placementDemands']:
@@ -60,31 +59,29 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_
     reservation_policies = [x for x in reservation_policy_list if len(x) > 0]
     reservation_groups = list_flatten(reservation_policies)
     req_info = request_json['requestInfo']
-    model_name = request_json['serviceInfo']['serviceName']
-    service_type = model_name
+    request_type = req_info.get('requestType', None)
+    service_type = request_json['serviceInfo']['serviceName']
     service_info = local_config.get('service_info', {}).get(service_type, {})
     order_info = {}
     if 'orderInfo' in request_json["placementInfo"]:
         order_info = json.loads(request_json["placementInfo"]["orderInfo"])
-    request_type = req_info.get('requestType', None)
+
     subs_com_site_id = ""
     if 'subscriberInfo' in request_json['placementInfo']: 
         subs_com_site_id = request_json['placementInfo']['subscriberInfo'].get('subscriberCommonSiteId', "")
-    rendered_req = None
-    if service_type == 'vCPE':
-        rendered_req = templ.render(
-            requestType=request_type,
-            chosenComplex=subs_com_site_id,
-            demand_list=demand_list,
-            policy_groups=policy_groups,
-            optimization_policies=optimization_policy_list,
-            name=req_info['requestId'],
-            timeout=req_info['timeout'],
-            limit=req_info['numSolutions'],
-            serviceType=service_type,
-            serviceInstance=request_json['serviceInfo']['serviceInstanceId'],
-            provStatus=prov_status,
-            chosenRegion=order_info.get('requestParameters', {}).get('lcpCloudRegionId'),
-            json=json)
+    rendered_req = templ.render(
+        requestType=request_type,
+        chosenComplex=subs_com_site_id,
+        demand_list=demand_list,
+        policy_groups=policy_groups,
+        optimization_policies=optimization_policy_list,
+        name=req_info['requestId'],
+        timeout=req_info['timeout'],
+        limit=req_info['numSolutions'],
+        service_type=request_json['serviceInfo']['serviceName'],
+        service_id=request_json['serviceInfo']['serviceInstanceId'],
+        provStatus="",
+        chosenRegion=order_info.get('requestParameters', {}).get('lcpCloudRegionId'),
+        json=json)
     json_payload = json.dumps(json.loads(rendered_req))  # need this because template's JSON is ugly!
     return json_payload
diff --git a/osdf/optimizers/placementopt/conductor/translation.py b/osdf/optimizers/placementopt/conductor/translation.py
index 9111c81..91706b7 100644
--- a/osdf/optimizers/placementopt/conductor/translation.py
+++ b/osdf/optimizers/placementopt/conductor/translation.py
@@ -37,8 +37,8 @@ def gen_optimization_policy(vnf_list, optimization_policy):
 
         for attr in content['objectiveParameter']['parameterAttributes']:
             parameter = attr['parameter'] if attr['parameter'] == "cloud_version" else attr['parameter']+"_between"
-            for res in attr['resource']:
-                vnf = get_matching_vnf(res, vnf_list)
+            vnfs = get_matching_vnfs(attr['resources'], vnf_list)
+            for vnf in vnfs:
                 value = [vnf] if attr['parameter'] == "cloud_version" else [attr['customerLocationInfo'], vnf]
                 parameter_list.append({
                     attr['operator']: [attr['weight'], {parameter: value}]
@@ -66,7 +66,7 @@ def get_matching_vnfs(resources, vnf_list, match_type="intersection"):
     :return: List of matching VNFs
     """
     if match_type == "all":  # don't bother with any comparisons
-        return resources
+        return resources if set(resources) <= set(vnf_list) else None
     common_vnfs = set(vnf_list) & set(resources)
     if match_type == "intersection":  # specifically requested intersection
         return list(common_vnfs)
@@ -87,8 +87,8 @@ def gen_policy_instance(vnf_list, resource_policy, match_type="intersection", rt
     related_policies = []
     for policy in resource_policy:
         pc = policy['content']
-        demands = get_matching_vnfs(pc['resourceInstanceType'], vnf_list, match_type=match_type)
-        resource = {pc['identity']: {'type': pc['type'], 'demands': demands}}
+        demands = get_matching_vnfs(pc['resources'], vnf_list, match_type=match_type)
+        resource = {pc['identity']: {'type': pc['policyType'], 'demands': demands}}
 
         if rtype:
             resource[pc['identity']]['properties'] = {'controller': pc[rtype]['controller'],
@@ -127,10 +127,10 @@ def gen_distance_to_location_policy(vnf_list, distance_to_location_policy):
     """Get policies governing distance-to-location for VNFs in order to populate the Conductor API call"""
     cur_policies, related_policies = gen_policy_instance(vnf_list, distance_to_location_policy, rtype=None)
     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
-        properties = p_main['content']['distanceToLocationProperty']
-        pcp_d = properties['distanceCondition']
+        properties = p_main['content']['distanceProperties']
+        pcp_d = properties['distance']
         p_new[p_main['content']['identity']]['properties'] = {
-            'distance': text_to_symbol[pcp_d['operator']] + " " + pcp_d['value'].lower(),
+            'distance': pcp_d['operator'] + " " + pcp_d['value'].lower() + " " + pcp_d['unit'].lower(),
             'location': properties['locationInfo']
         }
     return cur_policies
@@ -150,9 +150,9 @@ def gen_attribute_policy(vnf_list, attribute_policy):
 
 def gen_zone_policy(vnf_list, zone_policy):
     """Get zone policies in order to populate the Conductor API call"""
-    cur_policies, related_policies = gen_policy_instance(vnf_list, zone_policy, rtype=None)
+    cur_policies, related_policies = gen_policy_instance(vnf_list, zone_policy, match_type="all", rtype=None)
     for p_new, p_main in zip(cur_policies, related_policies):  # add additional fields to each policy
-        pmz = p_main['content']['zoneProperty']
+        pmz = p_main['content']['affinityProperty']
         p_new[p_main['content']['identity']]['properties'] = {'category': pmz['category'], 'qualifier': pmz['qualifier']}
     return cur_policies
 
@@ -179,9 +179,9 @@ def get_candidates_demands(demand):
 def get_policy_properties(demand, policies):
     """Get policy_properties for cases where there is a match with the demand"""
     for policy in policies:
-        if not set(policy['content'].get('resourceInstanceType', [])) & set(demand['resourceModuleName']):
+        if demand['resourceModuleName'] not in set(policy['content'].get('resources', [])):
             continue  # no match for this policy
-        for policy_property in policy['content']['property']:
+        for policy_property in policy['content']['vnfProperties']:
             yield policy_property
 
 
diff --git a/osdf/templates/conductor_interface.json b/osdf/templates/conductor_interface.json
index 2b48647..64f9c4f 100755
--- a/osdf/templates/conductor_interface.json
+++ b/osdf/templates/conductor_interface.json
@@ -4,52 +4,20 @@
   "timeout": {{ timeout }},

   "limit": {{ limit }},

   "template": {

-    "conductor_template_version": "2018-02-01",

+    "homing_template_version": "2017-10-10",

     "parameters": {

-	    "REQUEST_TYPE": "{{ requestType }}",

-        "CHOSEN_REGION": "{{ chosenRegion }}",

-        "LATITUDE": "{{ latitude }}",

-        "LONGITUDE": "{{ longitude }}",

-        {% if serviceType == 'DHV' %}

-        "E2EVPNKEY": "{{ e2eVpnKey }}",

-        "UCPEHOST": "{{ ucpeHostName }}",

-        "EFFECTIVE_BANDWIDTH": "{{ effectiveBandwidth }}",

-        "WAN_PORT1_UP": "{{ ipsec_bw_up }}",

-        "WAN_PORT1_DOWN": "{{ ipsec_bw_down }}",

-        "WAN_PORT2_UP": "{{ ipsec2_bw_up }}",

-        "WAN_PORT2_DOWN": "{{ ipsec2_bw_down }}",

-        {% endif %}

-        {% if serviceType != 'DHV' %}

-        "GW_TENANT_ID": "{{ gwTenantId }}",

-        "PORTAL_TENANT_ID": "{{ portalTenantId }}",

-        {% endif %}

-        "CHOSEN_COMPLEX": "{{ chosenComplex }}",

-        {% if serviceType == 'ADIOD' or serviceType == 'VPE' %}

-        "BANDWIDTH": "{{ bandwidth }}",

-        "UNIT": "{{ bandwidth_unit }}",

-        {% endif %}

-        "SERVICE_INST": "{{ serviceInstance }}",

-    	"PROV_STATUS": {{ json.dumps(provStatus) }}

+	    "service_name": "{{ service_name }}",

+        "service_id": "{{ service_id }}",

+        "customer_lat": "{{ latitude }}",

+        "customer_long": "{{ longitude }}",

+        "REQUIRED_MEM": "{{ memory }}",

+        "REQUIRED_DISK": "{{ storage }}"

     },

     "locations": {

-        {% if serviceType == 'DHV' %}

         "customer_loc": {

-            "host_name": { "get_param": "UCPEHOST" }

+            "latitude": { "get_param": "customer_lat" },

+            "longitude": { "get_param": "customer_long" }

         }

-        {% elif serviceType == 'ADIOD' %}

-        "customer_pref_location": {

-            "clli_code": { "get_param": "CHOSEN_COMPLEX" }

-        }

-        {% elif serviceType == 'NETBOND' %}

-        "peering_point": {

-            "latitude": { "get_param": "LATITUDE" },

-            "longitude": { "get_param": "LONGITUDE" }

-        }

-        {% else %}

-        "customer_pref_loc": {

-            "clli_code": { "get_param": "CHOSEN_COMPLEX" }

-        }

-        {% endif %}

     },

     "demands": {{ json.dumps(demand_list) }},

     {% set comma_main = joiner(",") %}

@@ -61,14 +29,6 @@
         {% endfor %}

       {% endfor %}

     },

-    "reservation": {

-      {% set comma=joiner(",") %}

-      {% for elem in reservation_groups %} {{ comma() }}

-        {% for key, value in elem.items() %}

-          "{{key}}": {{ json.dumps(value) }}

-        {% endfor %}

-      {% endfor %}

-    },

     "optimization": {

       {% set comma=joiner(",") %}

       {% for elem in optimization_policies %} {{ comma() }}