From 1a9638f5d5fc78f7e8be700e71b506fed3cc9d2d Mon Sep 17 00:00:00 2001 From: Sastry Isukapalli Date: Sat, 17 Mar 2018 02:40:15 -0400 Subject: New policies and required code changes Issue-ID: OPTFRA-146 Change-Id: I2261ae69f52b184cd7dcb7b86d5905538666a411 Signed-off-by: Sastry Isukapalli --- osdf/adapters/local_data/local_policies.py | 11 ++ osdf/adapters/policy/interface.py | 20 ++-- osdf/adapters/policy/utils.py | 8 +- osdf/adapters/sdc/constraint_handler.py | 81 ------------- osdf/adapters/sdc/sdc.py | 40 ------- osdf/models/api/placementRequest.py | 82 ++++++-------- osdf/models/api/placementResponse.py | 33 +++--- .../licenseopt/simple_license_allocation.py | 23 +--- .../placementopt/conductor/api_builder.py | 28 +---- .../optimizers/placementopt/conductor/conductor.py | 3 +- .../placementopt/conductor/remote_opt_processor.py | 4 +- .../placementopt/conductor/translation.py | 125 ++++++++++----------- osdf/utils/interfaces.py | 14 +++ 13 files changed, 170 insertions(+), 302 deletions(-) delete mode 100644 osdf/adapters/sdc/constraint_handler.py delete mode 100755 osdf/adapters/sdc/sdc.py (limited to 'osdf') diff --git a/osdf/adapters/local_data/local_policies.py b/osdf/adapters/local_data/local_policies.py index c63ae5a..e722fe2 100644 --- a/osdf/adapters/local_data/local_policies.py +++ b/osdf/adapters/local_data/local_policies.py @@ -18,6 +18,7 @@ import json import os +import re def get_local_policies(local_policy_folder, local_policy_list, policy_id_list=None): @@ -38,3 +39,13 @@ def get_local_policies(local_policy_folder, local_policy_list, policy_id_list=No with open(os.path.join(local_policy_folder, fname)) as fid: policies.append(json.load(fid)) return policies + + +def get_policy_names_from_file(fname_for_list_of_files): + """Get policy names; take care of comments, empty lines, etc""" + with open(fname_for_list_of_files) as fid: + return [ + re.sub(r'#.*$', '', x).strip() # remove inline comments and strip spaces + for x in fid + if not re.search(r'^#|^$', x.strip()) # remove blank or comments-only lines + ] diff --git a/osdf/adapters/policy/interface.py b/osdf/adapters/policy/interface.py index 4ddee15..288571f 100644 --- a/osdf/adapters/policy/interface.py +++ b/osdf/adapters/policy/interface.py @@ -110,9 +110,7 @@ def get_by_scope(rest_client, req, config_local, type_service): model_name = retrieve_node(req, pscope['service_name']) service_name = model_name - # service_name = data_mapping.get_request_service_type(req) - # if service_name is None: - # service_name = data_mapping.get_service_type(model_name) + scope = pscope['scope_{}'.format(service_name.lower())] subscriber_role, prov_status = get_subscriber_role(rest_client, req, pmain, service_name, scope) policy_type_list = pmain['policy_type_{}'.format(service_name.lower())] @@ -134,9 +132,7 @@ def remote_api(req_json, osdf_config, service_type="placement"): :param service_type: the type of service to call: "placement", "scheduling" :return: all related policies and provStatus retrieved from Subscriber policy """ -# if not req_json[service_type + "Info"]['policyId']: -# return [] - + prov_status = None config = osdf_config.deployment uid, passwd = config['policyPlatformUsername'], config['policyPlatformPassword'] pcuid, pcpasswd = config['policyClientUsername'], config['policyClientPassword'] @@ -168,16 +164,18 @@ def local_policies_location(req_json, osdf_config, service_type): :param service_type: placement supported for now, but can be any other service :return: a tuple (folder, file_list) or None """ - lp = osdf_config.core.get('osdf_hacks', {}).get('local_policies', {}) + lp = osdf_config.core.get('osdf_temp', {}).get('local_policies', {}) if lp.get('global_disabled'): return None # short-circuit to disable all local policies if lp.get('local_{}_policies_enabled'.format(service_type)): if service_type == "scheduling": return lp.get('{}_policy_dir'.format(service_type)), lp.get('{}_policy_files'.format(service_type)) else: - model_name = retrieve_node(req_json, osdf_config.core['policy_info'][service_type]['policy_scope']['service_name']) - service_name = data_mapping.get_service_type(model_name) - return lp.get('{}_policy_dir_{}'.format(service_type, service_name.lower())), lp.get('{}_policy_files_{}'.format(service_type, service_name.lower())) + required_node = osdf_config.core['policy_info'][service_type]['policy_scope']['service_name'] + model_name = retrieve_node(req_json, required_node) + service_name = model_name # TODO: data_mapping.get_service_type(model_name) + return lp.get('{}_policy_dir_{}'.format(service_type, service_name.lower())), \ + lp.get('{}_policy_files_{}'.format(service_type, service_name.lower())) return None @@ -199,6 +197,6 @@ def get_policies(request_json, service_type): to_filter = request_json[service_type + "Info"]['policyId'] policies = get_local_policies(local_info[0], local_info[1], to_filter) else: - policies, prov_status= remote_api(request_json, osdf_config, service_type) + policies, prov_status = remote_api(request_json, osdf_config, service_type) return policies, prov_status diff --git a/osdf/adapters/policy/utils.py b/osdf/adapters/policy/utils.py index a006f12..27885a5 100644 --- a/osdf/adapters/policy/utils.py +++ b/osdf/adapters/policy/utils.py @@ -31,10 +31,12 @@ def group_policies(flat_policies): filter_policies = defaultdict(list) policy_name = [] for policy in flat_policies: - policy_type = policy['content']['type'] + policy_type = policy['content'].get('policyType') + if not policy_type: + continue if policy_type not in aggregated_policies: aggregated_policies[policy_type] = defaultdict(list) - for resource in policy['content']['policyScope']['resourceInstanceType']: + for resource in policy['content'].get('resourceInstanceType', []): aggregated_policies[policy_type][resource].append(policy) for policy_type in aggregated_policies: for resource in aggregated_policies[policy_type]: @@ -42,7 +44,7 @@ def group_policies(flat_policies): aggregated_policies[policy_type][resource].sort(key=lambda x: x['priority'], reverse=True) policy = aggregated_policies[policy_type][resource][0] if policy['policyName'] not in policy_name: - filter_policies[policy['content']['type']].append(policy) + filter_policies[policy['content']['policyType']].append(policy) policy_name.append(policy['policyName']) return filter_policies diff --git a/osdf/adapters/sdc/constraint_handler.py b/osdf/adapters/sdc/constraint_handler.py deleted file mode 100644 index 2aae9a0..0000000 --- a/osdf/adapters/sdc/constraint_handler.py +++ /dev/null @@ -1,81 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) 2015-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -from osdf.config.base import osdf_config -from osdf.utils.programming_utils import dot_notation - -ns = {'p': 'http://xmlns.onap.org/sdc/license-model/1.0'} -config_local = osdf_config.core - - -def choose_license(license_artifacts, order_info, service_type): - entitlement_pool_uuids = [] - license_key_group_uuids = [] - - for license_artifact in license_artifacts: - for feature in license_artifact.findall('./p:feature-group-list/', ns): - for entitlement in feature.findall('./p:entitlement-pool-list/', ns): - if is_valid(entitlement, order_info, service_type): - entitlement_pool_uuid = entitlement.find('p:entitlement-pool-uuid', ns).text - entitlement_pool_uuids.append(entitlement_pool_uuid) - for license_key_group in feature.findall('./p:license-key-group-list/', ns): - if is_valid(license_key_group, order_info, service_type): - license_key_group_uuid = license_key_group.find('p:license-key-group-uuid', ns).text - license_key_group_uuids.append(license_key_group_uuid) - return entitlement_pool_uuids, license_key_group_uuids - - -# element is expected to be a license-key-group or entitlement-pool -# if these elements diverge at a later date this method should be refactored -def is_valid(element, order_info, service_type): - for limit in element.findall('./p:sp-limits/p:limit', ns): - # description = limit.find('p:description', ns).text - metric_value = limit.find('p:values', ns).text - metric = limit.find('p:metric', ns).text - try: - order_value = dot_notation(order_info, config_local['service_info'][service_type][metric]) - # print("The order has the value %s for the metric %s and the limit specifies the value %s. The limit has the description %s." % (order_value, metric, metric_value, description)) - if isinstance(order_value, list): # it is possible a list is returned, for example a list of vnfs for vCPE - for arr_value in order_value: - if str(metric_value) != str(arr_value): - return False - else: - if str(metric_value) != str(order_value): - return False - except KeyError: - return False - # vendor limits - for limit in element.findall('./p:vendor-limits/p:limit', ns): - # description = limit.find('p:description', ns).text - metric_value = limit.find('p:values', ns).text - metric = limit.find('p:metric', ns).text - try: - order_value = dot_notation(order_info, config_local['service_info'][service_type][metric]) - if isinstance(order_value, list): # it is possible a list is returned, for example a list of vnfs for vCPE - for arr_value in order_value: - if str(metric_value) != str(arr_value): - return False - else: - if str(metric_value) != str(order_value): - return False - # print("The order has the value %s for the metric %s and the limit specifies the value %s. The limit has the description %s." % (order_value, metric, metric_value, description)) - - except KeyError: - return False - return True - diff --git a/osdf/adapters/sdc/sdc.py b/osdf/adapters/sdc/sdc.py deleted file mode 100755 index 43932ba..0000000 --- a/osdf/adapters/sdc/sdc.py +++ /dev/null @@ -1,40 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) 2015-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -from osdf.utils.interfaces import RestClient -import xml.etree.ElementTree as ET - -def request(model_version_id, request_id, config): - """Get all of the license artifacts from SDC using service_resource_id and model_version_id - :param model_version_id: model_version_id - :param request_id: request_id - :return: license artifacts from SDC - """ - base_url = config['sdcUrl'] - uid, passwd = config['sdcUsername'], config['sdcPassword'] - headers = {"CSP_UID": config['sdcMechId'], "X-ONAP-InstanceID": "osdf"} - rc = RestClient(userid=uid, passwd=passwd, headers=headers, method="GET", req_id=request_id) - resource_data = rc.request(base_url + '/resources/{}/metadata'.format(model_version_id)) - - artifact_ids = [x['artifactURL'].split("/resources/")[-1] # get the part after /resources/ - for x in resource_data.get('artifacts', []) if x.get('artifactType') == "VF_LICENSE"] - artifact_urls = [base_url + '/resources/' + str(artifact_id) for artifact_id in artifact_ids] - licenses = [] - for x in artifact_urls: - licenses.append(ET.fromstring(rc.request(x, asjson=False))) - return licenses diff --git a/osdf/models/api/placementRequest.py b/osdf/models/api/placementRequest.py index df5f931..aa71eac 100644 --- a/osdf/models/api/placementRequest.py +++ b/osdf/models/api/placementRequest.py @@ -26,60 +26,61 @@ class RequestInfo(OSDFModel): transactionId = StringType(required=True) requestId = StringType(required=True) callbackUrl = URLType(required=True) + callbackHeader = DictType(BaseType) sourceId = StringType(required=True) - optimizers = ListType(StringType(required=True)) + requestType = StringType(required=True) numSolutions = IntType() + optimizers = ListType(StringType(required=True)) timeout = IntType() - requestType = StringType() -class CandidateInfo(OSDFModel): +class Candidates(OSDFModel): """Preferred candidate for a resource (sent as part of a request from client)""" - candidateType = StringType(required=True) - candidates = ListType(StringType(required=True)) + identifierType = StringType(required=True) + identifiers = ListType(StringType(required=True)) + cloudOwner = StringType() -class ResourceModelInfo(OSDFModel): +class ModelMetaData(OSDFModel): """Model information for a specific resource""" - modelCustomizationId = StringType(required=True) modelInvariantId = StringType(required=True) + modelVersionId = StringType(required=True) modelName = StringType() - modelVersion = StringType() - modelVersionId = StringType() modelType = StringType() + modelVersion = StringType() + modelCustomizationName = StringType(required=True) -class ExistingLicenseInfo(OSDFModel): - entitlementPoolUUID = ListType(StringType()) - licenseKeyGroupUUID = ListType(StringType()) +class LicenseModel(OSDFModel): + entitlementPoolUUID = ListType(StringType(required=True)) + licenseKeyGroupUUID = ListType(StringType(required=True)) -class LicenseDemand(OSDFModel): - resourceInstanceType = StringType(required=True) - serviceResourceId = StringType(required=True) +class LicenseDemands(OSDFModel): resourceModuleName = StringType(required=True) - resourceModelInfo = ModelType(ResourceModelInfo) - existingLicense = ModelType(ExistingLicenseInfo) + serviceResourceId = StringType(required=True) + resourceModelInfo = ModelType(ModelMetaData, required=True) + existingLicenses = ModelType(LicenseModel) + + +class LicenseInfo(OSDFModel): + licenseDemands = ListType(ModelType(LicenseDemands)) class PlacementDemand(OSDFModel): - resourceInstanceType = StringType(required=True) - serviceResourceId = StringType(required=True) resourceModuleName = StringType(required=True) - exclusionCandidateInfo = ListType(ModelType(CandidateInfo)) - requiredCandidateInfo = ListType(ModelType(CandidateInfo)) - resourceModelInfo = ModelType(ResourceModelInfo) - tenantId = StringType(required=True) - tenantName = StringType() - -class ExistingPlacementInfo(OSDFModel): - serviceInstanceId = StringType(required=True) + serviceResourceId = StringType(required=True) + tenantId = StringType() + resourceModelInfo = ModelType(ModelMetaData, required=True) + existingCandidates = ListType(ModelType(Candidates)) + excludedCandidates = ListType(ModelType(Candidates)) + requiredCandidates = ListType(ModelType(Candidates)) -class DemandInfo(OSDFModel): - """Requested resources (sent as part of a request from client)""" - placementDemand = ListType(ModelType(PlacementDemand)) - licenseDemand = ListType(ModelType(LicenseDemand)) +class ServiceInfo(OSDFModel): + serviceInstanceId = StringType(required=True) + modelInfo = ModelType(ModelMetaData, required=True) + serviceName = StringType(required=True) class SubscriberInfo(OSDFModel): @@ -89,27 +90,16 @@ class SubscriberInfo(OSDFModel): subscriberCommonSiteId = StringType() -class ServiceModelInfo(OSDFModel): - """ASDC Service model information""" - modelType = StringType(required=True) - modelInvariantId = StringType(required=True) - modelVersionId = StringType(required=True) - modelName = StringType(required=True) - modelVersion = StringType(required=True) - - class PlacementInfo(OSDFModel): """Information specific to placement optimization""" - serviceModelInfo = ModelType(ServiceModelInfo, required=True) - subscriberInfo = ModelType(SubscriberInfo, required=True) - demandInfo = ModelType(DemandInfo, required=True) requestParameters = DictType(BaseType) - policyId = ListType(StringType()) - serviceInstanceId = StringType(required=True) - existingPlacement = ModelType(ExistingPlacementInfo) + placementDemands = ListType(ModelType(PlacementDemand), min_size=1) + subscriberInfo = ModelType(SubscriberInfo) class PlacementAPI(OSDFModel): """Request for placement optimization (specific to optimization and additional metadata""" requestInfo = ModelType(RequestInfo, required=True) placementInfo = ModelType(PlacementInfo, required=True) + licenseInfo = ModelType(LicenseInfo) + serviceInfo = ModelType(ServiceInfo, required=True) diff --git a/osdf/models/api/placementResponse.py b/osdf/models/api/placementResponse.py index e9746d6..75d33f0 100644 --- a/osdf/models/api/placementResponse.py +++ b/osdf/models/api/placementResponse.py @@ -26,32 +26,39 @@ from schematics.types.compound import ModelType, ListType class LicenseSolution(OSDFModel): serviceResourceId = StringType(required=True) resourceModuleName = StringType(required=True) - entitlementPoolList = ListType(StringType(required=True)) - licenseKeyGroupList = ListType(StringType(required=True)) + entitlementPoolUUID = ListType(StringType(required=True)) + licenseKeyGroupUUID = ListType(StringType(required=True)) + entitlementPoolInvariantUUID = ListType(StringType(required=True)) + licenseKeyGroupInvariantUUID = ListType(StringType(required=True)) + + +class Candidates(OSDFModel): + """Preferred candidate for a resource (sent as part of a request from client)""" + identifierType = StringType(required=True) + identifiers = ListType(StringType(required=True)) + cloudOwner = StringType() class AssignmentInfo(OSDFModel): - variableName = StringType(required=True) - variableValue = StringType(required=True) + key = StringType(required=True) + value = StringType(required=True) class PlacementSolution(OSDFModel): serviceResourceId = StringType(required=True) resourceModuleName = StringType(required=True) - inventoryType = StringType(required=True) - serviceInstanceId = StringType() - cloudRegionId = StringType() + solution = ModelType(Candidates, required=True) assignmentInfo = ListType(ModelType(AssignmentInfo)) -class SolutionInfo(OSDFModel): - placement = ListType(ModelType(PlacementSolution), min_size=1) - license = ListType(ModelType(LicenseSolution), min_size=1) +class Solution(OSDFModel): + placementSolutions = ListType(ListType(ModelType(PlacementSolution), min_size=1)) + licenseSolutions = ListType(ModelType(LicenseSolution), min_size=1) class PlacementResponse(OSDFModel): transactionId = StringType(required=True) requestId = StringType(required=True) - requestState = StringType(required=True) - statusMessage = StringType(required=True) - solutionInfo = ModelType(SolutionInfo) + requestStatus = StringType(required=True) + statusMessage = StringType() + solutions = ModelType(Solution, required=True) diff --git a/osdf/optimizers/licenseopt/simple_license_allocation.py b/osdf/optimizers/licenseopt/simple_license_allocation.py index beafbe4..ad9acfc 100644 --- a/osdf/optimizers/licenseopt/simple_license_allocation.py +++ b/osdf/optimizers/licenseopt/simple_license_allocation.py @@ -16,13 +16,6 @@ # ------------------------------------------------------------------------- # -import json - -from requests import RequestException -from osdf.adapters.sdc import sdc, constraint_handler -from osdf.logging.osdf_logging import audit_log, metrics_log, MH -from osdf.config.base import osdf_config - def license_optim(request_json): """ @@ -32,25 +25,17 @@ def license_optim(request_json): :return: A tuple of licensekey-group-uuid-list and entitlement-group-uuid-list """ req_id = request_json["requestInfo"]["requestId"] - config = osdf_config.deployment - model_name = request_json['placementInfo']['serviceModelInfo']['modelName'] - # service_name = data_mapping.get_service_type(model_name) + model_name = request_json.get('placementInfo', {}).get('serviceInfo', {}).get('modelInfo', {}).get('modelName') service_name = model_name license_info = [] - order_info = json.loads(request_json["placementInfo"]["requestParameters"]) - if service_name == 'VPE': - data_mapping.normalize_user_params(order_info) - for licenseDemand in request_json['placementInfo']['demandInfo']['licenseDemand']: - metrics_log.info(MH.requesting("sdc", req_id)) - license_artifacts = sdc.request(licenseDemand['resourceModelInfo']['modelVersionId'],request_json["requestInfo"]["requestId"], config) - entitlement_pool_uuids, license_key_group_uuids = constraint_handler.choose_license(license_artifacts,order_info, service_name) + for licenseDemand in request_json.get('placementInfo', {}).get('demandInfo', {}).get('licenseDemands', []): license_info.append( {'serviceResourceId': licenseDemand['serviceResourceId'], 'resourceModuleName': licenseDemand['resourceModuleName'], - 'entitlementPoolList': entitlement_pool_uuids, - 'licenseKeyGroupList': license_key_group_uuids + 'entitlementPoolList': "NOT SUPPORTED", + 'licenseKeyGroupList': "NOT SUPPORTED" }) return license_info diff --git a/osdf/optimizers/placementopt/conductor/api_builder.py b/osdf/optimizers/placementopt/conductor/api_builder.py index 0a874f7..d8a2083 100644 --- a/osdf/optimizers/placementopt/conductor/api_builder.py +++ b/osdf/optimizers/placementopt/conductor/api_builder.py @@ -18,7 +18,6 @@ import copy import json -# from osdf.utils import data_mapping from jinja2 import Template from osdf.utils.programming_utils import list_flatten, dot_notation import osdf.optimizers.placementopt.conductor.translation as tr @@ -39,10 +38,10 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_ gp = group_policies(flat_policies) demand_vnf_name_list = [] - for placementDemand in request_json['placementInfo']['demandInfo']['placementDemand']: + for placementDemand in request_json['placementInfo']['placementDemands']: demand_vnf_name_list.append(placementDemand['resourceModuleName']) - demand_list = tr.gen_demands(request_json['placementInfo']['demandInfo'], gp['vnfPolicy']) + demand_list = tr.gen_demands(request_json, gp['vnfPolicy']) attribute_policy_list = tr.gen_attribute_policy(demand_vnf_name_list, gp['attribute']) distance_to_location_policy_list = tr.gen_distance_to_location_policy( demand_vnf_name_list, gp['distance_to_location']) @@ -60,7 +59,7 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_ reservation_policies = [x for x in reservation_policy_list if len(x) > 0] reservation_groups = list_flatten(reservation_policies) req_info = request_json['requestInfo'] - model_name = request_json['placementInfo']['serviceModelInfo']['modelName'] + model_name = request_json['serviceInfo']['serviceName'] service_type = model_name # service_type = data_mapping.get_service_type(model_name) service_info = local_config.get('service_info', {}).get(service_type, {}) @@ -71,6 +70,7 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_ subs_com_site_id = "" if 'subscriberInfo' in request_json['placementInfo']: subs_com_site_id = request_json['placementInfo']['subscriberInfo'].get('subscriberCommonSiteId', "") + rendered_req = None if service_type == 'vCPE': # data_mapping.normalize_user_params(order_info) rendered_req = templ.render( @@ -83,28 +83,10 @@ def conductor_api_builder(request_json, flat_policies: list, local_config, prov_ timeout=req_info['timeout'], limit=req_info['numSolutions'], serviceType=service_type, - serviceInstance=request_json['placementInfo']['serviceInstanceId'], + serviceInstance=request_json['serviceInfo']['serviceInstanceId'], provStatus = prov_status, chosenRegion=order_info.get('requestParameters',{}).get('lcpCloudRegionId'), json=json) - elif service_type == 'UNKNOWN': - rendered_req = templ.render( - requestType=request_type, - chosenComplex=subs_com_site_id, - demand_list=demand_list, - policy_groups=policy_groups, - reservation_groups=reservation_groups, - optimization_policies=optimization_policy_list, - name=req_info['requestId'], - timeout=req_info['timeout'], - limit=req_info['numSolutions'], - serviceType=service_type, - serviceInstance=request_json['placementInfo']['serviceInstanceId'], - provStatus = prov_status, - # process order data - bandwidth=dot_notation(order_info, service_info['bandwidth']), - bandwidth_unit=dot_notation(order_info, service_info['bandwidth_units']), - json=json) json_payload = json.dumps(json.loads(rendered_req)) # need this because template's JSON is ugly! return json_payload diff --git a/osdf/optimizers/placementopt/conductor/conductor.py b/osdf/optimizers/placementopt/conductor/conductor.py index bdc7f17..32a4728 100644 --- a/osdf/optimizers/placementopt/conductor/conductor.py +++ b/osdf/optimizers/placementopt/conductor/conductor.py @@ -78,7 +78,8 @@ def request(req_object, osdf_config, grouped_policies, prov_status): new_url = resp['plans'][0]['links'][0][0]['href'] # TODO: check why a list of lists if total_time >= max_timeout: - raise BusinessException("Conductor could not provide a solution within {} seconds, this transaction is timing out".format(max_timeout)) + raise BusinessException("Conductor could not provide a solution within {} seconds," + "this transaction is timing out".format(max_timeout)) time.sleep(ping_wait_time) ctr += 1 debug_log.debug("Attempt number {} url {}; prior status={}".format(ctr, new_url, resp['plans'][0]['status'])) diff --git a/osdf/optimizers/placementopt/conductor/remote_opt_processor.py b/osdf/optimizers/placementopt/conductor/remote_opt_processor.py index f753a70..957547b 100644 --- a/osdf/optimizers/placementopt/conductor/remote_opt_processor.py +++ b/osdf/optimizers/placementopt/conductor/remote_opt_processor.py @@ -43,11 +43,11 @@ def process_placement_opt(request_json, policies, osdf_config, prov_status): metrics_log.info(MH.inside_worker_thread(req_id)) license_info = None - if 'licenseDemand' in request_json['placementInfo']['demandInfo']: + if request_json.get('licenseInfo', {}).get('licenseDemands'): license_info = license_optim(request_json) # Conductor only handles placement, only call Conductor if placementDemands exist - if 'placementDemand' in request_json['placementInfo']['demandInfo']: + if request_json.get('licenseInfo', {}).get('licenseDemands'): metrics_log.info(MH.requesting("placement/conductor", req_id)) placement_response = conductor.request(request_json, osdf_config, policies, prov_status) if license_info: # Attach license solution if it exists diff --git a/osdf/optimizers/placementopt/conductor/translation.py b/osdf/optimizers/placementopt/conductor/translation.py index 262fa86..9111c81 100644 --- a/osdf/optimizers/placementopt/conductor/translation.py +++ b/osdf/optimizers/placementopt/conductor/translation.py @@ -15,16 +15,20 @@ # # ------------------------------------------------------------------------- # - +import copy import json +import yaml + from osdf.utils.data_conversion import text_to_symbol -# from osdf.utils import data_mapping + +policy_config_mapping = yaml.load(open('config/has_config.yaml')).get('policy_config_mapping') + def gen_optimization_policy(vnf_list, optimization_policy): """Generate optimization policy details to pass to Conductor :param vnf_list: List of vnf's to used in placement request - :param optimization_policy: optimization policy information provided in the incoming request - :return: List of optimization policies in a format required by Conductor + :param optimization_policy: optimization objective policy information provided in the incoming request + :return: List of optimization objective policies in a format required by Conductor """ optimization_policy_list = [] for policy in optimization_policy: @@ -47,7 +51,7 @@ def gen_optimization_policy(vnf_list, optimization_policy): def get_matching_vnf(resource, vnf_list): - + for vnf in vnf_list: if resource in vnf: return vnf @@ -57,20 +61,16 @@ def get_matching_vnf(resource, vnf_list): def get_matching_vnfs(resources, vnf_list, match_type="intersection"): """Get a list of matching VNFs from the list of resources :param resources: - :param vnf_list: List of vnf's to used in placement request + :param vnf_list: List of vnfs to used in placement request :param match_type: "intersection" or "all" or "any" (any => send all_vnfs if there is any intersection) :return: List of matching VNFs """ - common_vnfs = [] - for vnf in vnf_list: - for resource in resources: - if resource in vnf: - common_vnfs.append(vnf) - if match_type == "intersection": # specifically requested intersection - return common_vnfs - elif common_vnfs or match_type == "all": # ("any" and common) OR "all" + if match_type == "all": # don't bother with any comparisons return resources - return None + common_vnfs = set(vnf_list) & set(resources) + if match_type == "intersection": # specifically requested intersection + return list(common_vnfs) + return resources if common_vnfs else None # "any" match => all resources to be returned def gen_policy_instance(vnf_list, resource_policy, match_type="intersection", rtype=None): @@ -141,22 +141,11 @@ def gen_attribute_policy(vnf_list, attribute_policy): cur_policies, related_policies = gen_policy_instance(vnf_list, attribute_policy, rtype=None) for p_new, p_main in zip(cur_policies, related_policies): # add additional fields to each policy properties = p_main['content']['cloudAttributeProperty'] + attribute_mapping = policy_config_mapping['attributes'] # wanted attributes and mapping p_new[p_main['content']['identity']]['properties'] = { - 'evaluate': { - 'hypervisor': properties.get('hypervisor', ''), - 'cloud_version': properties.get('cloudVersion', ''), - 'cloud_type': properties.get('cloudType', ''), - 'dataplane': properties.get('dataPlane', ''), - 'network_roles': properties.get('networkRoles', ''), - 'complex': properties.get('complex', ''), - 'state': properties.get('state', ''), - 'country': properties.get('country', ''), - 'geo_region': properties.get('geoRegion', ''), - 'exclusivity_groups': properties.get('exclusivityGroups', ''), - 'replication_role': properties.get('replicationRole', '') - } + 'evaluate': dict((k, properties.get(attribute_mapping.get(k))) for k in attribute_mapping.keys()) } - return cur_policies + return cur_policies # cur_policies gets updated in place... def gen_zone_policy(vnf_list, zone_policy): @@ -168,39 +157,49 @@ def gen_zone_policy(vnf_list, zone_policy): return cur_policies +def get_augmented_policy_attributes(policy_property, demand): + """Get policy attributes and augment them using policy_config_mapping and demand information""" + attributes = copy.copy(policy_property['attributes']) + remapping = policy_config_mapping['remapping'] + extra = dict((x, demand['resourceModelInfo'][remapping[x]]) for x in attributes if x in remapping) + attributes.update(extra) + return attributes + + +def get_candidates_demands(demand): + """Get demands related to candidates; e.g. excluded/required""" + res = {} + for k, v in policy_config_mapping['candidates'].items(): + if k not in demand: + continue + res[v] = [{'inventory_type': x['candidateType'], 'candidate_id': x['candidates']} for x in demand[k]] + return res + + +def get_policy_properties(demand, policies): + """Get policy_properties for cases where there is a match with the demand""" + for policy in policies: + if not set(policy['content'].get('resourceInstanceType', [])) & set(demand['resourceModuleName']): + continue # no match for this policy + for policy_property in policy['content']['property']: + yield policy_property + + def get_demand_properties(demand, policies): """Get list demand properties objects (named tuples) from policy""" - def _get_candidates(candidate_info): - return [dict(inventory_type=x['candidateType'], candidate_id=x['candidates']) for x in candidate_info] - properties = [] - for policy in policies: - for resourceInstanceType in policy['content']['resourceInstanceType']: - if resourceInstanceType in demand['resourceModuleName']: - for x in policy['content']['property']: - property = dict(inventory_provider=x['inventoryProvider'], - inventory_type=x['inventoryType'], - service_resource_id=demand['serviceResourceId']) - if 'attributes' in x: - attributes = {} - for k,v in x['attributes'].items(): - # key=data_mapping.convert(k) - key = k - attributes[key] = v - if(key=="model-invariant-id"): - attributes[key]=demand['resourceModelInfo']['modelInvariantId'] - elif(key=="model-version-id"): - attributes[key]=demand['resourceModelInfo']['modelVersionId'] - property.update({"attributes": attributes}) - if x['inventoryType'] == "cloud": - property['region'] = {'get_param': "CHOSEN_REGION"} - if 'exclusionCandidateInfo' in demand: - property['excluded_candidates'] = _get_candidates(demand['exclusionCandidateInfo']) - if 'requiredCandidateInfo' in demand: - property['required_candidates'] = _get_candidates(demand['requiredCandidateInfo']) - properties.append(property) - if len(properties) == 0: - properties.append(dict(customer_id="", service_type="", inventory_provider="", inventory_type="")) - return properties + demand_properties = [] + for policy_property in get_policy_properties(demand, policies): + prop = dict(inventory_provider=policy_property['inventoryProvider'], + inventory_type=policy_property['inventoryType'], + service_resource_id=demand['serviceResourceId']) + if 'attributes' in policy_property: + prop['attributes'] = get_augmented_policy_attributes(policy_property, demand) + for k1, v1, k2, v2 in policy_config_mapping['extra_fields']: + if k1 == v1: + prop[k2] = v2 + prop.update(get_candidates_demands(demand)) # for excluded and partial-rehoming cases + demand_properties.append(prop) + return demand_properties def gen_demands(req_json, vnf_policies): @@ -210,7 +209,7 @@ def gen_demands(req_json, vnf_policies): :return: list of demand parameters to populate the Conductor API call """ demand_dictionary = {} - for placementDemand in req_json['placementDemand']: - demand_dictionary.update({placementDemand['resourceModuleName']: get_demand_properties(placementDemand, vnf_policies)}) - + for demand in req_json['placementInfo']['placementDemands']: + demand_dictionary.update( + {demand['resourceModuleName']: get_demand_properties(demand, vnf_policies)}) return demand_dictionary diff --git a/osdf/utils/interfaces.py b/osdf/utils/interfaces.py index 7a0e3a9..a869d6d 100644 --- a/osdf/utils/interfaces.py +++ b/osdf/utils/interfaces.py @@ -16,7 +16,9 @@ # ------------------------------------------------------------------------- # +import json import requests +import yaml from osdf.config.base import osdf_config, creds_prefixes from osdf.logging.osdf_logging import MH, debug_log @@ -35,6 +37,18 @@ def get_rest_client(request_json, service): return RestClient(url=callback_url, userid=c_userid, passwd=c_passwd) +def json_from_file(file_name): + """Read a policy from a file""" + with open(file_name) as fid: + return json.load(fid) + + +def yaml_from_file(file_name): + """Read a policy from a file""" + with open(file_name) as fid: + return yaml.load(fid) + + class RestClient(object): """Simple REST Client that supports get/post and basic auth""" -- cgit 1.2.3-korg