aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVikas Varma <vikas.varma@att.com>2020-09-18 19:33:06 +0000
committerGerrit Code Review <gerrit@onap.org>2020-09-18 19:33:06 +0000
commitf9b3575cae2b521ba8c6b6b30b15c89bd8a1cb48 (patch)
tree96ad4aebcb7cb459dac02c43de37e9a15509991c
parenta55cbabaef4975d5007363e59d4560cb30a855e3 (diff)
parent716ad1f32fc10181d2826d1f345f5a8b81bd0106 (diff)
Merge "Add ML based optimization to PCI opt"
-rw-r--r--apps/pci/optimizers/solver/ml_model.py69
-rw-r--r--apps/pci/optimizers/solver/optimizer.py20
-rw-r--r--config/common_config.yaml10
-rwxr-xr-xconfig/osdf_config.yaml8
-rw-r--r--osdf/adapters/dcae/des.py51
-rw-r--r--test/adapters/dcae/des_response.json47
-rw-r--r--test/adapters/dcae/test_des.py69
-rw-r--r--test/apps/pci_optimization/des_result.json86
-rw-r--r--test/apps/pci_optimization/test_ml_model.py87
-rw-r--r--test/config/common_config.yaml12
-rwxr-xr-xtest/config/osdf_config.yaml9
-rw-r--r--test/functest/simulators/simulated-config/common_config.yaml10
12 files changed, 473 insertions, 5 deletions
diff --git a/apps/pci/optimizers/solver/ml_model.py b/apps/pci/optimizers/solver/ml_model.py
new file mode 100644
index 0000000..3e12387
--- /dev/null
+++ b/apps/pci/optimizers/solver/ml_model.py
@@ -0,0 +1,69 @@
+# -------------------------------------------------------------------------
+# Copyright (C) 2020 Wipro Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+from osdf.adapters.dcae import des
+from osdf.adapters.dcae.des import DESException
+from osdf.config.base import osdf_config
+from osdf.logging.osdf_logging import error_log
+
+
+class MlModel(object):
+ def __init__(self):
+ self.config = osdf_config.core['PCI']
+
+ def get_additional_inputs(self, dzn_data, network_cell_info):
+ """Add/update additional info to the existing models.
+
+ The method returns nothing. Instead, it modifies the dzn_data
+ :params: dzn_data: map with data for the optimization
+ """
+ self.compute_ml_model(dzn_data, network_cell_info)
+
+ def compute_ml_model(self, dzn_data, network_cell_info):
+ average_ho_threshold = self.config['ML']['average_ho_threshold']
+ latest_ho_threshold = self.config['ML']['latest_ho_threshold']
+
+ fixed_cells = set()
+ for cell in network_cell_info['cell_list']:
+ cell_id = cell['cell_id']
+ average_ho, latest_ho = self.get_ho_details(cell['cell_id'])
+ if average_ho > average_ho_threshold or latest_ho > latest_ho_threshold:
+ fixed_cells.add(cell_id)
+
+ fixed_cells.update(dzn_data.get('PCI_UNCHANGEABLE_CELLS', []))
+ dzn_data['PCI_UNCHANGEABLE_CELLS'] = list(fixed_cells)
+
+ def get_ho_details(self, cell_id):
+ service_id = self.config['DES']['service_id']
+ request_data = self.config['DES']['filter']
+ request_data['cell_id'] = cell_id
+ try:
+ result = des.extract_data(service_id, request_data)
+ except DESException as e:
+ error_log.error("Error while calling DES {}".format(e))
+ return 0, 0
+
+ if not result:
+ return 0, 0
+
+ ho_list = []
+ for pm_data in result:
+ ho = sum([int(meas['hashMap']['InterEnbOutAtt_X2HO']) for meas in pm_data['additionalMeasurements']])
+ ho_list.append(ho)
+
+ return sum(ho_list) / len(ho_list), ho_list[0]
diff --git a/apps/pci/optimizers/solver/optimizer.py b/apps/pci/optimizers/solver/optimizer.py
index 940f9f7..13298ed 100644
--- a/apps/pci/optimizers/solver/optimizer.py
+++ b/apps/pci/optimizers/solver/optimizer.py
@@ -1,5 +1,6 @@
# -------------------------------------------------------------------------
# Copyright (c) 2018 AT&T Intellectual Property
+# Copyright (C) 2020 Wipro Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -16,17 +17,21 @@
# -------------------------------------------------------------------------
#
+from collections import defaultdict
import itertools
import os
-from collections import defaultdict
import pymzn
-from .pci_utils import get_id,mapping
+from apps.pci.optimizers.solver.ml_model import MlModel
+from apps.pci.optimizers.solver.pci_utils import get_id
+from apps.pci.optimizers.solver.pci_utils import mapping
+from osdf.config.base import osdf_config
BASE_DIR = os.path.dirname(__file__)
cell_id_mapping = dict()
id_cell_mapping = dict()
+
def pci_optimize(network_cell_info, cell_info_list, request_json):
global cell_id_mapping, id_cell_mapping
cell_id_mapping, id_cell_mapping = mapping(network_cell_info)
@@ -37,10 +42,16 @@ def pci_optimize(network_cell_info, cell_info_list, request_json):
ignorable_links = get_ignorable_links(network_cell_info, request_json)
anr_flag = is_anr(request_json)
- dzn_data = build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag, original_pcis, unchangeable_pcis)
+ dzn_data = build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag,
+ original_pcis, unchangeable_pcis)
+
+ ml_enabled = osdf_config.core['PCI']['ml_enabled']
+ if ml_enabled:
+ MlModel().get_additional_inputs(dzn_data, network_cell_info)
return build_pci_solution(dzn_data, ignorable_links, anr_flag)
+
def get_ids_of_fixed_pci_cells(fixed_pci_list):
fixed_pci_ids = set()
for cell in fixed_pci_list:
@@ -83,7 +94,8 @@ def build_pci_solution(dzn_data, ignorable_links, anr_flag):
return solution
-def build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag,original_pcis, unchangeable_pcis):
+def build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag, original_pcis,
+ unchangeable_pcis):
dzn_data = {
'NUM_NODES': len(cell_info_list),
'NUM_PCIS': len(cell_info_list),
diff --git a/config/common_config.yaml b/config/common_config.yaml
index 0f2dc96..f010e44 100644
--- a/config/common_config.yaml
+++ b/config/common_config.yaml
@@ -104,3 +104,13 @@ policy_info:
default: # if no explicit service related information is needed
policy_fetch: by_name
policy_scope: none
+
+PCI:
+ ML:
+ average_ho_threshold: 10000
+ latest_ho_threshold: 500
+ DES:
+ service_id: ho_metric
+ filter:
+ interval: 10
+ ml_enabled: false \ No newline at end of file
diff --git a/config/osdf_config.yaml b/config/osdf_config.yaml
index 4207b34..1800ce5 100755
--- a/config/osdf_config.yaml
+++ b/config/osdf_config.yaml
@@ -58,6 +58,14 @@ aaiGetControllersUrl: /aai/v19/external-system/esr-thirdparty-sdnc-list
controllerQueryUrl: /aai/v19/query?format=resource
aaiGetInterDomainLinksUrl: /aai/v19/network/logical-links?link-type=inter-domain&operational-status=up
+#DES api
+desUrl: https://des.url:9000
+desApiPath: /datalake/v1/exposure/
+desHeaders:
+ Accept: application/json
+ Content-Type: application/json
+desUsername:
+desPassword:
pciHMSUsername: test
pciHMSPassword: passwd
diff --git a/osdf/adapters/dcae/des.py b/osdf/adapters/dcae/des.py
new file mode 100644
index 0000000..57d0371
--- /dev/null
+++ b/osdf/adapters/dcae/des.py
@@ -0,0 +1,51 @@
+# -------------------------------------------------------------------------
+# Copyright (C) 2020 Wipro Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from osdf.config.base import osdf_config
+
+
+class DESException(Exception):
+ pass
+
+
+def extract_data(service_id, request_data):
+ """Extracts data from the data lake via DES.
+
+ param: service_id: kpi data identifier
+ param: request_data: data to send
+ param: osdf_config: osdf config to retrieve api info
+ """
+
+ config = osdf_config.deployment
+ user, password = config['desUsername'], config['desPassword']
+ auth = HTTPBasicAuth(user, password)
+ headers = config["desHeaders"]
+ req_url = config["desUrl"] + config["desApiPath"] + service_id
+
+ try:
+ response = requests.post(req_url, data=request_data, headers=headers, auth=auth, verify=False)
+ except requests.RequestException as e:
+ raise DESException("Request exception was encountered {}".format(e))
+
+ if response.status_code == 200:
+ return response.json().get("result")
+ else:
+ raise DESException("Response code other than 200. Response code: {}".format(response.status_code))
diff --git a/test/adapters/dcae/des_response.json b/test/adapters/dcae/des_response.json
new file mode 100644
index 0000000..c8595eb
--- /dev/null
+++ b/test/adapters/dcae/des_response.json
@@ -0,0 +1,47 @@
+{
+ "result": [
+ {
+ "additionalMeasurements": [
+ {
+ "hashMap":{
+ "networkId":"plmnid1",
+ "InterEnbOutAtt_X2HO":"300",
+ "InterEnbOutSucc_X2HO":"290"
+ },
+ "name":"Chn0004"
+ },
+ {
+ "hashMap":{
+ "InterEnbOutAtt_X2HO":"250",
+ "InterEnbOutSucc_X2HO":"170"
+ },
+ "name":"Chn0001"
+ }
+ ]
+ },
+ {
+ "additionalMeasurements": [
+ {
+ "hashMap":{
+ "networkId":"plmnid1",
+ "InterEnbOutAtt_X2HO":"300",
+ "InterEnbOutSucc_X2HO":"290"
+ },
+ "name":"Chn0004"
+ },
+ {
+ "hashMap":{
+ "InterEnbOutAtt_X2HO":"250",
+ "InterEnbOutSucc_X2HO":"170"
+ },
+ "name":"Chn0001"
+ }
+ ]
+ }
+ ],
+ "request": {
+ "cell_id": "Chn0002",
+ "interval": 2
+ },
+ "result_count": 2
+} \ No newline at end of file
diff --git a/test/adapters/dcae/test_des.py b/test/adapters/dcae/test_des.py
new file mode 100644
index 0000000..6daa29b
--- /dev/null
+++ b/test/adapters/dcae/test_des.py
@@ -0,0 +1,69 @@
+# -------------------------------------------------------------------------
+# Copyright (C) 2020 Wipro Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+import mock
+from mock import patch
+from requests import RequestException
+import unittest
+from osdf.adapters.dcae import des
+from osdf.adapters.dcae.des import DESException
+import osdf.config.loader as config_loader
+from osdf.utils.interfaces import json_from_file
+from osdf.utils.programming_utils import DotDict
+
+
+class TestDes(unittest.TestCase):
+
+ def setUp(self):
+ self.config_spec = {
+ "deployment": "config/osdf_config.yaml",
+ "core": "config/common_config.yaml"
+ }
+ self.osdf_config = DotDict(config_loader.all_configs(**self.config_spec))
+
+ def tearDown(self):
+ pass
+
+ def test_extract_data(self):
+ response_file = 'test/adapters/dcae/des_response.json'
+ response_json = json_from_file(response_file)
+
+ des_config = self.osdf_config.core['PCI']['DES']
+ service_id = des_config['service_id']
+ data = des_config['filter']
+ expected = response_json['result']
+ response = mock.MagicMock()
+ response.status_code = 200
+ response.ok = True
+ response.json.return_value = response_json
+ self.patcher_req = patch('requests.post', return_value=response)
+ self.Mock_req = self.patcher_req.start()
+ self.assertEqual(expected, des.extract_data(service_id, data))
+ self.patcher_req.stop()
+
+ response = mock.MagicMock()
+ response.status_code = 404
+ self.patcher_req = patch('requests.post', return_value=response)
+ self.Mock_req = self.patcher_req.start()
+ self.assertRaises(DESException, des.extract_data, service_id, data)
+ self.patcher_req.stop()
+
+ self.patcher_req = patch('requests.post', side_effect=RequestException("error"))
+ self.Mock_req = self.patcher_req.start()
+ self.assertRaises(DESException, des.extract_data, service_id, data)
+ self.patcher_req.stop()
diff --git a/test/apps/pci_optimization/des_result.json b/test/apps/pci_optimization/des_result.json
new file mode 100644
index 0000000..2083ade
--- /dev/null
+++ b/test/apps/pci_optimization/des_result.json
@@ -0,0 +1,86 @@
+[
+ [
+ {
+ "additionalMeasurements": [
+ {
+ "hashMap":{
+ "networkId":"plmnid1",
+ "InterEnbOutAtt_X2HO":"300",
+ "InterEnbOutSucc_X2HO":"290"
+ },
+ "name":"Chn0004"
+ },
+ {
+ "hashMap":{
+ "InterEnbOutAtt_X2HO":"1000",
+ "InterEnbOutSucc_X2HO":"170"
+ },
+ "name":"Chn0001"
+ }
+
+ ]
+ },
+ {
+ "additionalMeasurements": [
+ {
+ "hashMap":{
+ "networkId":"plmnid1",
+ "InterEnbOutAtt_X2HO":"300",
+ "InterEnbOutSucc_X2HO":"290"
+ },
+ "name":"Chn0004"
+ },
+ {
+ "hashMap":{
+ "InterEnbOutAtt_X2HO":"250",
+ "InterEnbOutSucc_X2HO":"170"
+ },
+ "name":"Chn0001"
+ }
+
+ ]
+ }
+ ],
+ [
+ {
+ "additionalMeasurements": [
+ {
+ "hashMap":{
+ "networkId":"plmnid1",
+ "InterEnbOutAtt_X2HO":"200",
+ "InterEnbOutSucc_X2HO":"290"
+ },
+ "name":"Chn0007"
+ },
+ {
+ "hashMap":{
+ "InterEnbOutAtt_X2HO":"250",
+ "InterEnbOutSucc_X2HO":"170"
+ },
+ "name":"Chn0005"
+ }
+
+ ]
+ },
+ {
+ "additionalMeasurements": [
+ {
+ "hashMap":{
+ "networkId":"plmnid1",
+ "InterEnbOutAtt_X2HO":"150",
+ "InterEnbOutSucc_X2HO":"290"
+ },
+ "name":"Chn0007"
+ },
+ {
+ "hashMap":{
+ "InterEnbOutAtt_X2HO":"250",
+ "InterEnbOutSucc_X2HO":"170"
+ },
+ "name":"Chn0005"
+ }
+
+ ]
+ }
+ ]
+] \ No newline at end of file
diff --git a/test/apps/pci_optimization/test_ml_model.py b/test/apps/pci_optimization/test_ml_model.py
new file mode 100644
index 0000000..9c617a9
--- /dev/null
+++ b/test/apps/pci_optimization/test_ml_model.py
@@ -0,0 +1,87 @@
+# -------------------------------------------------------------------------
+# Copyright (C) 2020 Wipro Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+import copy
+from mock import patch
+import unittest
+from apps.pci.optimizers.solver.ml_model import MlModel
+from osdf.adapters.dcae.des import DESException
+import osdf.config.loader as config_loader
+from osdf.utils.interfaces import json_from_file
+from osdf.utils.programming_utils import DotDict
+
+
+class TestMlModel(unittest.TestCase):
+ def setUp(self):
+ self.config_spec = {
+ "deployment": "config/osdf_config.yaml",
+ "core": "config/common_config.yaml"
+ }
+ self.osdf_config = DotDict(config_loader.all_configs(**self.config_spec))
+
+ def tearDown(self):
+ pass
+
+ def test_ml_model(self):
+ des_result_file = 'test/apps/pci_optimization/des_result.json'
+ results = json_from_file(des_result_file)
+
+ dzn_data = {
+ 'NUM_NODES': 4,
+ 'NUM_PCIS': 4,
+ 'NUM_NEIGHBORS': 4,
+ 'NEIGHBORS': [],
+ 'NUM_SECOND_LEVEL_NEIGHBORS': 1,
+ 'SECOND_LEVEL_NEIGHBORS': [],
+ 'PCI_UNCHANGEABLE_CELLS': [],
+ 'ORIGINAL_PCIS': []
+ }
+
+ network_cell_info = {
+ 'cell_list': [
+ {
+ 'cell_id': 'Chn0001',
+ 'id': 1,
+ 'nbr_list': []
+ },
+ {
+ 'cell_id': 'Chn0002',
+ 'id': 2,
+ 'nbr_list': []
+ }
+ ]
+ }
+ self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', side_effect=results)
+ self.Mock_req = self.patcher_req.start()
+ mlmodel = MlModel()
+ mlmodel.get_additional_inputs(dzn_data, network_cell_info)
+ self.assertEqual(['Chn0001'], dzn_data['PCI_UNCHANGEABLE_CELLS'])
+ self.patcher_req.stop()
+
+ dzn_data['PCI_UNCHANGEABLE_CELLS'] = []
+ self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', side_effect=DESException('error'))
+ self.Mock_req = self.patcher_req.start()
+ mlmodel.get_additional_inputs(dzn_data, network_cell_info)
+ self.assertEqual([], dzn_data['PCI_UNCHANGEABLE_CELLS'])
+ self.patcher_req.stop()
+
+ self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', return_value=[])
+ self.Mock_req = self.patcher_req.start()
+ mlmodel.get_additional_inputs(dzn_data, network_cell_info)
+ self.assertEqual([], dzn_data['PCI_UNCHANGEABLE_CELLS'])
+ self.patcher_req.stop()
diff --git a/test/config/common_config.yaml b/test/config/common_config.yaml
index a3ef82e..560d707 100644
--- a/test/config/common_config.yaml
+++ b/test/config/common_config.yaml
@@ -61,4 +61,14 @@ policy_info:
- get_param: subscriber_role
default: # if no explicit service related information is needed
policy_fetch: by_name
- policy_scope: none \ No newline at end of file
+ policy_scope: none
+
+PCI:
+ ML:
+ average_ho_threshold: 10000
+ latest_ho_threshold: 500
+ DES:
+ service_id: ho_metric
+ filter:
+ interval: 10
+ ml_enabled: false
diff --git a/test/config/osdf_config.yaml b/test/config/osdf_config.yaml
index 2ed8b88..ef73d4c 100755
--- a/test/config/osdf_config.yaml
+++ b/test/config/osdf_config.yaml
@@ -70,3 +70,12 @@ osdfPCIOptPassword: PCI-OSDF-PASSWD
aaiUrl: "https://aai.url:30233"
aaiServiceInstanceUrl : "/aai/v20/nodes/service-instances/service-instance/"
+
+#DES api
+desUrl: https://des.url:9000
+desApiPath: /datalake/v1/exposure/
+desHeaders:
+ Accept: application/json
+ Content-Type: application/json
+desUsername:
+desPassword:
diff --git a/test/functest/simulators/simulated-config/common_config.yaml b/test/functest/simulators/simulated-config/common_config.yaml
index 1249dc0..36c639f 100644
--- a/test/functest/simulators/simulated-config/common_config.yaml
+++ b/test/functest/simulators/simulated-config/common_config.yaml
@@ -62,3 +62,13 @@ policy_info:
default: # if no explicit service related information is needed
policy_fetch: by_name
policy_scope: none
+
+PCI:
+ ML:
+ average_ho_threshold: 10000
+ latest_ho_threshold: 500
+ DES:
+ service_id: ho_metric
+ filter:
+ interval: 10
+ ml_enabled: false