diff options
| -rw-r--r-- | apps/pci/optimizers/solver/ml_model.py | 69 | ||||
| -rw-r--r-- | apps/pci/optimizers/solver/optimizer.py | 20 | ||||
| -rw-r--r-- | config/common_config.yaml | 10 | ||||
| -rwxr-xr-x | config/osdf_config.yaml | 8 | ||||
| -rw-r--r-- | osdf/adapters/dcae/des.py | 51 | ||||
| -rw-r--r-- | test/adapters/dcae/des_response.json | 47 | ||||
| -rw-r--r-- | test/adapters/dcae/test_des.py | 69 | ||||
| -rw-r--r-- | test/apps/pci_optimization/des_result.json | 86 | ||||
| -rw-r--r-- | test/apps/pci_optimization/test_ml_model.py | 87 | ||||
| -rw-r--r-- | test/config/common_config.yaml | 12 | ||||
| -rwxr-xr-x | test/config/osdf_config.yaml | 9 | ||||
| -rw-r--r-- | test/functest/simulators/simulated-config/common_config.yaml | 10 |
12 files changed, 473 insertions, 5 deletions
diff --git a/apps/pci/optimizers/solver/ml_model.py b/apps/pci/optimizers/solver/ml_model.py new file mode 100644 index 0000000..3e12387 --- /dev/null +++ b/apps/pci/optimizers/solver/ml_model.py @@ -0,0 +1,69 @@ +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from osdf.adapters.dcae import des +from osdf.adapters.dcae.des import DESException +from osdf.config.base import osdf_config +from osdf.logging.osdf_logging import error_log + + +class MlModel(object): + def __init__(self): + self.config = osdf_config.core['PCI'] + + def get_additional_inputs(self, dzn_data, network_cell_info): + """Add/update additional info to the existing models. + + The method returns nothing. Instead, it modifies the dzn_data + :params: dzn_data: map with data for the optimization + """ + self.compute_ml_model(dzn_data, network_cell_info) + + def compute_ml_model(self, dzn_data, network_cell_info): + average_ho_threshold = self.config['ML']['average_ho_threshold'] + latest_ho_threshold = self.config['ML']['latest_ho_threshold'] + + fixed_cells = set() + for cell in network_cell_info['cell_list']: + cell_id = cell['cell_id'] + average_ho, latest_ho = self.get_ho_details(cell['cell_id']) + if average_ho > average_ho_threshold or latest_ho > latest_ho_threshold: + fixed_cells.add(cell_id) + + fixed_cells.update(dzn_data.get('PCI_UNCHANGEABLE_CELLS', [])) + dzn_data['PCI_UNCHANGEABLE_CELLS'] = list(fixed_cells) + + def get_ho_details(self, cell_id): + service_id = self.config['DES']['service_id'] + request_data = self.config['DES']['filter'] + request_data['cell_id'] = cell_id + try: + result = des.extract_data(service_id, request_data) + except DESException as e: + error_log.error("Error while calling DES {}".format(e)) + return 0, 0 + + if not result: + return 0, 0 + + ho_list = [] + for pm_data in result: + ho = sum([int(meas['hashMap']['InterEnbOutAtt_X2HO']) for meas in pm_data['additionalMeasurements']]) + ho_list.append(ho) + + return sum(ho_list) / len(ho_list), ho_list[0] diff --git a/apps/pci/optimizers/solver/optimizer.py b/apps/pci/optimizers/solver/optimizer.py index 940f9f7..13298ed 100644 --- a/apps/pci/optimizers/solver/optimizer.py +++ b/apps/pci/optimizers/solver/optimizer.py @@ -1,5 +1,6 @@ # ------------------------------------------------------------------------- # Copyright (c) 2018 AT&T Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,17 +17,21 @@ # ------------------------------------------------------------------------- # +from collections import defaultdict import itertools import os -from collections import defaultdict import pymzn -from .pci_utils import get_id,mapping +from apps.pci.optimizers.solver.ml_model import MlModel +from apps.pci.optimizers.solver.pci_utils import get_id +from apps.pci.optimizers.solver.pci_utils import mapping +from osdf.config.base import osdf_config BASE_DIR = os.path.dirname(__file__) cell_id_mapping = dict() id_cell_mapping = dict() + def pci_optimize(network_cell_info, cell_info_list, request_json): global cell_id_mapping, id_cell_mapping cell_id_mapping, id_cell_mapping = mapping(network_cell_info) @@ -37,10 +42,16 @@ def pci_optimize(network_cell_info, cell_info_list, request_json): ignorable_links = get_ignorable_links(network_cell_info, request_json) anr_flag = is_anr(request_json) - dzn_data = build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag, original_pcis, unchangeable_pcis) + dzn_data = build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag, + original_pcis, unchangeable_pcis) + + ml_enabled = osdf_config.core['PCI']['ml_enabled'] + if ml_enabled: + MlModel().get_additional_inputs(dzn_data, network_cell_info) return build_pci_solution(dzn_data, ignorable_links, anr_flag) + def get_ids_of_fixed_pci_cells(fixed_pci_list): fixed_pci_ids = set() for cell in fixed_pci_list: @@ -83,7 +94,8 @@ def build_pci_solution(dzn_data, ignorable_links, anr_flag): return solution -def build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag,original_pcis, unchangeable_pcis): +def build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag, original_pcis, + unchangeable_pcis): dzn_data = { 'NUM_NODES': len(cell_info_list), 'NUM_PCIS': len(cell_info_list), diff --git a/config/common_config.yaml b/config/common_config.yaml index 0f2dc96..f010e44 100644 --- a/config/common_config.yaml +++ b/config/common_config.yaml @@ -104,3 +104,13 @@ policy_info: default: # if no explicit service related information is needed policy_fetch: by_name policy_scope: none + +PCI: + ML: + average_ho_threshold: 10000 + latest_ho_threshold: 500 + DES: + service_id: ho_metric + filter: + interval: 10 + ml_enabled: false
\ No newline at end of file diff --git a/config/osdf_config.yaml b/config/osdf_config.yaml index 4207b34..1800ce5 100755 --- a/config/osdf_config.yaml +++ b/config/osdf_config.yaml @@ -58,6 +58,14 @@ aaiGetControllersUrl: /aai/v19/external-system/esr-thirdparty-sdnc-list controllerQueryUrl: /aai/v19/query?format=resource aaiGetInterDomainLinksUrl: /aai/v19/network/logical-links?link-type=inter-domain&operational-status=up +#DES api +desUrl: https://des.url:9000 +desApiPath: /datalake/v1/exposure/ +desHeaders: + Accept: application/json + Content-Type: application/json +desUsername: +desPassword: pciHMSUsername: test pciHMSPassword: passwd diff --git a/osdf/adapters/dcae/des.py b/osdf/adapters/dcae/des.py new file mode 100644 index 0000000..57d0371 --- /dev/null +++ b/osdf/adapters/dcae/des.py @@ -0,0 +1,51 @@ +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import requests +from requests.auth import HTTPBasicAuth + +from osdf.config.base import osdf_config + + +class DESException(Exception): + pass + + +def extract_data(service_id, request_data): + """Extracts data from the data lake via DES. + + param: service_id: kpi data identifier + param: request_data: data to send + param: osdf_config: osdf config to retrieve api info + """ + + config = osdf_config.deployment + user, password = config['desUsername'], config['desPassword'] + auth = HTTPBasicAuth(user, password) + headers = config["desHeaders"] + req_url = config["desUrl"] + config["desApiPath"] + service_id + + try: + response = requests.post(req_url, data=request_data, headers=headers, auth=auth, verify=False) + except requests.RequestException as e: + raise DESException("Request exception was encountered {}".format(e)) + + if response.status_code == 200: + return response.json().get("result") + else: + raise DESException("Response code other than 200. Response code: {}".format(response.status_code)) diff --git a/test/adapters/dcae/des_response.json b/test/adapters/dcae/des_response.json new file mode 100644 index 0000000..c8595eb --- /dev/null +++ b/test/adapters/dcae/des_response.json @@ -0,0 +1,47 @@ +{ + "result": [ + { + "additionalMeasurements": [ + { + "hashMap":{ + "networkId":"plmnid1", + "InterEnbOutAtt_X2HO":"300", + "InterEnbOutSucc_X2HO":"290" + }, + "name":"Chn0004" + }, + { + "hashMap":{ + "InterEnbOutAtt_X2HO":"250", + "InterEnbOutSucc_X2HO":"170" + }, + "name":"Chn0001" + } + ] + }, + { + "additionalMeasurements": [ + { + "hashMap":{ + "networkId":"plmnid1", + "InterEnbOutAtt_X2HO":"300", + "InterEnbOutSucc_X2HO":"290" + }, + "name":"Chn0004" + }, + { + "hashMap":{ + "InterEnbOutAtt_X2HO":"250", + "InterEnbOutSucc_X2HO":"170" + }, + "name":"Chn0001" + } + ] + } + ], + "request": { + "cell_id": "Chn0002", + "interval": 2 + }, + "result_count": 2 +}
\ No newline at end of file diff --git a/test/adapters/dcae/test_des.py b/test/adapters/dcae/test_des.py new file mode 100644 index 0000000..6daa29b --- /dev/null +++ b/test/adapters/dcae/test_des.py @@ -0,0 +1,69 @@ +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import mock +from mock import patch +from requests import RequestException +import unittest +from osdf.adapters.dcae import des +from osdf.adapters.dcae.des import DESException +import osdf.config.loader as config_loader +from osdf.utils.interfaces import json_from_file +from osdf.utils.programming_utils import DotDict + + +class TestDes(unittest.TestCase): + + def setUp(self): + self.config_spec = { + "deployment": "config/osdf_config.yaml", + "core": "config/common_config.yaml" + } + self.osdf_config = DotDict(config_loader.all_configs(**self.config_spec)) + + def tearDown(self): + pass + + def test_extract_data(self): + response_file = 'test/adapters/dcae/des_response.json' + response_json = json_from_file(response_file) + + des_config = self.osdf_config.core['PCI']['DES'] + service_id = des_config['service_id'] + data = des_config['filter'] + expected = response_json['result'] + response = mock.MagicMock() + response.status_code = 200 + response.ok = True + response.json.return_value = response_json + self.patcher_req = patch('requests.post', return_value=response) + self.Mock_req = self.patcher_req.start() + self.assertEqual(expected, des.extract_data(service_id, data)) + self.patcher_req.stop() + + response = mock.MagicMock() + response.status_code = 404 + self.patcher_req = patch('requests.post', return_value=response) + self.Mock_req = self.patcher_req.start() + self.assertRaises(DESException, des.extract_data, service_id, data) + self.patcher_req.stop() + + self.patcher_req = patch('requests.post', side_effect=RequestException("error")) + self.Mock_req = self.patcher_req.start() + self.assertRaises(DESException, des.extract_data, service_id, data) + self.patcher_req.stop() diff --git a/test/apps/pci_optimization/des_result.json b/test/apps/pci_optimization/des_result.json new file mode 100644 index 0000000..2083ade --- /dev/null +++ b/test/apps/pci_optimization/des_result.json @@ -0,0 +1,86 @@ +[ + [ + { + "additionalMeasurements": [ + { + "hashMap":{ + "networkId":"plmnid1", + "InterEnbOutAtt_X2HO":"300", + "InterEnbOutSucc_X2HO":"290" + }, + "name":"Chn0004" + }, + { + "hashMap":{ + "InterEnbOutAtt_X2HO":"1000", + "InterEnbOutSucc_X2HO":"170" + }, + "name":"Chn0001" + } + + ] + }, + { + "additionalMeasurements": [ + { + "hashMap":{ + "networkId":"plmnid1", + "InterEnbOutAtt_X2HO":"300", + "InterEnbOutSucc_X2HO":"290" + }, + "name":"Chn0004" + }, + { + "hashMap":{ + "InterEnbOutAtt_X2HO":"250", + "InterEnbOutSucc_X2HO":"170" + }, + "name":"Chn0001" + } + + ] + } + ], + [ + { + "additionalMeasurements": [ + { + "hashMap":{ + "networkId":"plmnid1", + "InterEnbOutAtt_X2HO":"200", + "InterEnbOutSucc_X2HO":"290" + }, + "name":"Chn0007" + }, + { + "hashMap":{ + "InterEnbOutAtt_X2HO":"250", + "InterEnbOutSucc_X2HO":"170" + }, + "name":"Chn0005" + } + + ] + }, + { + "additionalMeasurements": [ + { + "hashMap":{ + "networkId":"plmnid1", + "InterEnbOutAtt_X2HO":"150", + "InterEnbOutSucc_X2HO":"290" + }, + "name":"Chn0007" + }, + { + "hashMap":{ + "InterEnbOutAtt_X2HO":"250", + "InterEnbOutSucc_X2HO":"170" + }, + "name":"Chn0005" + } + + ] + } + ] +]
\ No newline at end of file diff --git a/test/apps/pci_optimization/test_ml_model.py b/test/apps/pci_optimization/test_ml_model.py new file mode 100644 index 0000000..9c617a9 --- /dev/null +++ b/test/apps/pci_optimization/test_ml_model.py @@ -0,0 +1,87 @@ +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import copy +from mock import patch +import unittest +from apps.pci.optimizers.solver.ml_model import MlModel +from osdf.adapters.dcae.des import DESException +import osdf.config.loader as config_loader +from osdf.utils.interfaces import json_from_file +from osdf.utils.programming_utils import DotDict + + +class TestMlModel(unittest.TestCase): + def setUp(self): + self.config_spec = { + "deployment": "config/osdf_config.yaml", + "core": "config/common_config.yaml" + } + self.osdf_config = DotDict(config_loader.all_configs(**self.config_spec)) + + def tearDown(self): + pass + + def test_ml_model(self): + des_result_file = 'test/apps/pci_optimization/des_result.json' + results = json_from_file(des_result_file) + + dzn_data = { + 'NUM_NODES': 4, + 'NUM_PCIS': 4, + 'NUM_NEIGHBORS': 4, + 'NEIGHBORS': [], + 'NUM_SECOND_LEVEL_NEIGHBORS': 1, + 'SECOND_LEVEL_NEIGHBORS': [], + 'PCI_UNCHANGEABLE_CELLS': [], + 'ORIGINAL_PCIS': [] + } + + network_cell_info = { + 'cell_list': [ + { + 'cell_id': 'Chn0001', + 'id': 1, + 'nbr_list': [] + }, + { + 'cell_id': 'Chn0002', + 'id': 2, + 'nbr_list': [] + } + ] + } + self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', side_effect=results) + self.Mock_req = self.patcher_req.start() + mlmodel = MlModel() + mlmodel.get_additional_inputs(dzn_data, network_cell_info) + self.assertEqual(['Chn0001'], dzn_data['PCI_UNCHANGEABLE_CELLS']) + self.patcher_req.stop() + + dzn_data['PCI_UNCHANGEABLE_CELLS'] = [] + self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', side_effect=DESException('error')) + self.Mock_req = self.patcher_req.start() + mlmodel.get_additional_inputs(dzn_data, network_cell_info) + self.assertEqual([], dzn_data['PCI_UNCHANGEABLE_CELLS']) + self.patcher_req.stop() + + self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', return_value=[]) + self.Mock_req = self.patcher_req.start() + mlmodel.get_additional_inputs(dzn_data, network_cell_info) + self.assertEqual([], dzn_data['PCI_UNCHANGEABLE_CELLS']) + self.patcher_req.stop() diff --git a/test/config/common_config.yaml b/test/config/common_config.yaml index a3ef82e..560d707 100644 --- a/test/config/common_config.yaml +++ b/test/config/common_config.yaml @@ -61,4 +61,14 @@ policy_info: - get_param: subscriber_role default: # if no explicit service related information is needed policy_fetch: by_name - policy_scope: none
\ No newline at end of file + policy_scope: none + +PCI: + ML: + average_ho_threshold: 10000 + latest_ho_threshold: 500 + DES: + service_id: ho_metric + filter: + interval: 10 + ml_enabled: false diff --git a/test/config/osdf_config.yaml b/test/config/osdf_config.yaml index 2ed8b88..ef73d4c 100755 --- a/test/config/osdf_config.yaml +++ b/test/config/osdf_config.yaml @@ -70,3 +70,12 @@ osdfPCIOptPassword: PCI-OSDF-PASSWD aaiUrl: "https://aai.url:30233" aaiServiceInstanceUrl : "/aai/v20/nodes/service-instances/service-instance/" + +#DES api +desUrl: https://des.url:9000 +desApiPath: /datalake/v1/exposure/ +desHeaders: + Accept: application/json + Content-Type: application/json +desUsername: +desPassword: diff --git a/test/functest/simulators/simulated-config/common_config.yaml b/test/functest/simulators/simulated-config/common_config.yaml index 1249dc0..36c639f 100644 --- a/test/functest/simulators/simulated-config/common_config.yaml +++ b/test/functest/simulators/simulated-config/common_config.yaml @@ -62,3 +62,13 @@ policy_info: default: # if no explicit service related information is needed policy_fetch: by_name policy_scope: none + +PCI: + ML: + average_ho_threshold: 10000 + latest_ho_threshold: 500 + DES: + service_id: ho_metric + filter: + interval: 10 + ml_enabled: false |