Add ML based optimization to PCI opt

Add DCAE DES adapter to the adapters
Add a simple ML model in PCI opt app

Issue-ID: OPTFRA-769
Signed-off-by: krishnaa96 <krishna.moorthy6@wipro.com>
Change-Id: I144887e1be1ac6be4d27eeec22f9669c71f2c2bb

12 files changed, 473 insertions, 5 deletions

diff --git a/apps/pci/optimizers/solver/ml_model.py b/apps/pci/optimizers/solver/ml_model.py
new file mode 100644
index 0000000..3e12387
--- /dev/null
+++ b/apps/pci/optimizers/solver/ml_model.py
@@ -0,0 +1,69 @@
+# -------------------------------------------------------------------------
+#   Copyright (C) 2020 Wipro Limited.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+from osdf.adapters.dcae import des
+from osdf.adapters.dcae.des import DESException
+from osdf.config.base import osdf_config
+from osdf.logging.osdf_logging import error_log
+
+
+class MlModel(object):
+    def __init__(self):
+        self.config = osdf_config.core['PCI']
+
+    def get_additional_inputs(self, dzn_data, network_cell_info):
+        """Add/update additional info to the existing models.
+
+        The method returns nothing. Instead, it modifies the dzn_data
+        :params: dzn_data: map with data for the optimization
+        """
+        self.compute_ml_model(dzn_data, network_cell_info)
+
+    def compute_ml_model(self, dzn_data, network_cell_info):
+        average_ho_threshold = self.config['ML']['average_ho_threshold']
+        latest_ho_threshold = self.config['ML']['latest_ho_threshold']
+
+        fixed_cells = set()
+        for cell in network_cell_info['cell_list']:
+            cell_id = cell['cell_id']
+            average_ho, latest_ho = self.get_ho_details(cell['cell_id'])
+            if average_ho > average_ho_threshold or latest_ho > latest_ho_threshold:
+                fixed_cells.add(cell_id)
+
+        fixed_cells.update(dzn_data.get('PCI_UNCHANGEABLE_CELLS', []))
+        dzn_data['PCI_UNCHANGEABLE_CELLS'] = list(fixed_cells)
+
+    def get_ho_details(self, cell_id):
+        service_id = self.config['DES']['service_id']
+        request_data = self.config['DES']['filter']
+        request_data['cell_id'] = cell_id
+        try:
+            result = des.extract_data(service_id, request_data)
+        except DESException as e:
+            error_log.error("Error while calling DES {}".format(e))
+            return 0, 0
+
+        if not result:
+            return 0, 0
+
+        ho_list = []
+        for pm_data in result:
+            ho = sum([int(meas['hashMap']['InterEnbOutAtt_X2HO']) for meas in pm_data['additionalMeasurements']])
+            ho_list.append(ho)
+
+        return sum(ho_list) / len(ho_list), ho_list[0]
diff --git a/apps/pci/optimizers/solver/optimizer.py b/apps/pci/optimizers/solver/optimizer.py
index 940f9f7..13298ed 100644
--- a/apps/pci/optimizers/solver/optimizer.py
+++ b/apps/pci/optimizers/solver/optimizer.py
@@ -1,5 +1,6 @@
 # -------------------------------------------------------------------------
 #   Copyright (c) 2018 AT&T Intellectual Property
+#   Copyright (C) 2020 Wipro Limited.
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -16,17 +17,21 @@
 # -------------------------------------------------------------------------
 #
 
+from collections import defaultdict
 import itertools
 import os
-from collections import defaultdict
 import pymzn
 
-from .pci_utils import get_id,mapping
+from apps.pci.optimizers.solver.ml_model import MlModel
+from apps.pci.optimizers.solver.pci_utils import get_id
+from apps.pci.optimizers.solver.pci_utils import mapping
+from osdf.config.base import osdf_config
 
 BASE_DIR = os.path.dirname(__file__)
 cell_id_mapping = dict()
 id_cell_mapping = dict()
 
+
 def pci_optimize(network_cell_info, cell_info_list, request_json):
     global cell_id_mapping, id_cell_mapping
     cell_id_mapping, id_cell_mapping = mapping(network_cell_info)
@@ -37,10 +42,16 @@ def pci_optimize(network_cell_info, cell_info_list, request_json):
     ignorable_links = get_ignorable_links(network_cell_info, request_json)
     anr_flag = is_anr(request_json)
 
-    dzn_data = build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag, original_pcis, unchangeable_pcis)
+    dzn_data = build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag,
+                              original_pcis, unchangeable_pcis)
+
+    ml_enabled = osdf_config.core['PCI']['ml_enabled']
+    if ml_enabled:
+        MlModel().get_additional_inputs(dzn_data, network_cell_info)
 
     return build_pci_solution(dzn_data, ignorable_links, anr_flag)
 
+
 def get_ids_of_fixed_pci_cells(fixed_pci_list):
     fixed_pci_ids = set()
     for cell in fixed_pci_list:
@@ -83,7 +94,8 @@ def build_pci_solution(dzn_data, ignorable_links, anr_flag):
     return solution
 
 
-def build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag,original_pcis, unchangeable_pcis):
+def build_dzn_data(cell_info_list, ignorable_links, neighbor_edges, second_level_edges, anr_flag, original_pcis,
+                   unchangeable_pcis):
     dzn_data = {
         'NUM_NODES': len(cell_info_list),
         'NUM_PCIS': len(cell_info_list),
diff --git a/config/common_config.yaml b/config/common_config.yaml
index 0f2dc96..f010e44 100644
--- a/config/common_config.yaml
+++ b/config/common_config.yaml
@@ -104,3 +104,13 @@ policy_info:
     default:  # if no explicit service related information is needed
         policy_fetch: by_name
         policy_scope: none
+
+PCI:
+    ML:
+        average_ho_threshold: 10000
+        latest_ho_threshold: 500
+    DES:
+        service_id: ho_metric
+        filter:
+            interval: 10
+    ml_enabled: false
\ No newline at end of file
diff --git a/config/osdf_config.yaml b/config/osdf_config.yaml
index 4207b34..1800ce5 100755
--- a/config/osdf_config.yaml
+++ b/config/osdf_config.yaml
@@ -58,6 +58,14 @@ aaiGetControllersUrl: /aai/v19/external-system/esr-thirdparty-sdnc-list
 controllerQueryUrl: /aai/v19/query?format=resource
 aaiGetInterDomainLinksUrl: /aai/v19/network/logical-links?link-type=inter-domain&operational-status=up
 
+#DES api
+desUrl: https://des.url:9000
+desApiPath: /datalake/v1/exposure/
+desHeaders:
+  Accept: application/json
+  Content-Type: application/json
+desUsername:
+desPassword:
 
 pciHMSUsername: test
 pciHMSPassword: passwd
diff --git a/osdf/adapters/dcae/des.py b/osdf/adapters/dcae/des.py
new file mode 100644
index 0000000..57d0371
--- /dev/null
+++ b/osdf/adapters/dcae/des.py
@@ -0,0 +1,51 @@
+# -------------------------------------------------------------------------
+#   Copyright (C) 2020 Wipro Limited.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from osdf.config.base import osdf_config
+
+
+class DESException(Exception):
+    pass
+
+
+def extract_data(service_id, request_data):
+    """Extracts data from the data lake via DES.
+
+    param: service_id: kpi data identifier
+    param: request_data: data to send
+    param: osdf_config: osdf config to retrieve api info
+    """
+
+    config = osdf_config.deployment
+    user, password = config['desUsername'], config['desPassword']
+    auth = HTTPBasicAuth(user, password)
+    headers = config["desHeaders"]
+    req_url = config["desUrl"] + config["desApiPath"] + service_id
+
+    try:
+        response = requests.post(req_url, data=request_data, headers=headers, auth=auth, verify=False)
+    except requests.RequestException as e:
+        raise DESException("Request exception was encountered {}".format(e))
+
+    if response.status_code == 200:
+        return response.json().get("result")
+    else:
+        raise DESException("Response code other than 200. Response code: {}".format(response.status_code))
diff --git a/test/adapters/dcae/des_response.json b/test/adapters/dcae/des_response.json
new file mode 100644
index 0000000..c8595eb
--- /dev/null
+++ b/test/adapters/dcae/des_response.json
@@ -0,0 +1,47 @@
+{
+  "result": [
+    {
+      "additionalMeasurements": [
+    		{
+               "hashMap":{
+                  "networkId":"plmnid1",
+                  "InterEnbOutAtt_X2HO":"300",
+                  "InterEnbOutSucc_X2HO":"290"
+               },
+               "name":"Chn0004"
+            },
+            {
+               "hashMap":{
+                  "InterEnbOutAtt_X2HO":"250",
+                  "InterEnbOutSucc_X2HO":"170"
+               },
+               "name":"Chn0001"
+            }
+        ]
+    },
+    {
+      "additionalMeasurements": [
+    		{
+               "hashMap":{
+                  "networkId":"plmnid1",
+                  "InterEnbOutAtt_X2HO":"300",
+                  "InterEnbOutSucc_X2HO":"290"
+               },
+               "name":"Chn0004"
+            },
+            {
+               "hashMap":{
+                  "InterEnbOutAtt_X2HO":"250",
+                  "InterEnbOutSucc_X2HO":"170"
+               },
+               "name":"Chn0001"
+            }
+        ]
+    }
+  ],
+  "request": {
+    "cell_id": "Chn0002",
+    "interval": 2
+  },
+  "result_count": 2
+}
\ No newline at end of file
diff --git a/test/adapters/dcae/test_des.py b/test/adapters/dcae/test_des.py
new file mode 100644
index 0000000..6daa29b
--- /dev/null
+++ b/test/adapters/dcae/test_des.py
@@ -0,0 +1,69 @@
+# -------------------------------------------------------------------------
+#   Copyright (C) 2020 Wipro Limited.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+import mock
+from mock import patch
+from requests import RequestException
+import unittest
+from osdf.adapters.dcae import des
+from osdf.adapters.dcae.des import DESException
+import osdf.config.loader as config_loader
+from osdf.utils.interfaces import json_from_file
+from osdf.utils.programming_utils import DotDict
+
+
+class TestDes(unittest.TestCase):
+
+    def setUp(self):
+        self.config_spec = {
+            "deployment": "config/osdf_config.yaml",
+            "core": "config/common_config.yaml"
+        }
+        self.osdf_config = DotDict(config_loader.all_configs(**self.config_spec))
+
+    def tearDown(self):
+        pass
+
+    def test_extract_data(self):
+        response_file = 'test/adapters/dcae/des_response.json'
+        response_json = json_from_file(response_file)
+
+        des_config = self.osdf_config.core['PCI']['DES']
+        service_id = des_config['service_id']
+        data = des_config['filter']
+        expected = response_json['result']
+        response = mock.MagicMock()
+        response.status_code = 200
+        response.ok = True
+        response.json.return_value = response_json
+        self.patcher_req = patch('requests.post', return_value=response)
+        self.Mock_req = self.patcher_req.start()
+        self.assertEqual(expected, des.extract_data(service_id, data))
+        self.patcher_req.stop()
+
+        response = mock.MagicMock()
+        response.status_code = 404
+        self.patcher_req = patch('requests.post', return_value=response)
+        self.Mock_req = self.patcher_req.start()
+        self.assertRaises(DESException, des.extract_data, service_id, data)
+        self.patcher_req.stop()
+
+        self.patcher_req = patch('requests.post', side_effect=RequestException("error"))
+        self.Mock_req = self.patcher_req.start()
+        self.assertRaises(DESException, des.extract_data, service_id, data)
+        self.patcher_req.stop()
diff --git a/test/apps/pci_optimization/des_result.json b/test/apps/pci_optimization/des_result.json
new file mode 100644
index 0000000..2083ade
--- /dev/null
+++ b/test/apps/pci_optimization/des_result.json
@@ -0,0 +1,86 @@
+[
+  [
+  {
+    "additionalMeasurements": [
+    		{
+               "hashMap":{
+                  "networkId":"plmnid1",
+                  "InterEnbOutAtt_X2HO":"300",
+                  "InterEnbOutSucc_X2HO":"290"
+               },
+               "name":"Chn0004"
+            },
+            {
+               "hashMap":{
+                  "InterEnbOutAtt_X2HO":"1000",
+                  "InterEnbOutSucc_X2HO":"170"
+               },
+               "name":"Chn0001"
+            }
+
+        ]
+  },
+  {
+    "additionalMeasurements": [
+    		{
+               "hashMap":{
+                  "networkId":"plmnid1",
+                  "InterEnbOutAtt_X2HO":"300",
+                  "InterEnbOutSucc_X2HO":"290"
+               },
+               "name":"Chn0004"
+            },
+            {
+               "hashMap":{
+                  "InterEnbOutAtt_X2HO":"250",
+                  "InterEnbOutSucc_X2HO":"170"
+               },
+               "name":"Chn0001"
+            }
+
+        ]
+  }
+  ],
+  [
+  {
+    "additionalMeasurements": [
+    		{
+               "hashMap":{
+                  "networkId":"plmnid1",
+                  "InterEnbOutAtt_X2HO":"200",
+                  "InterEnbOutSucc_X2HO":"290"
+               },
+               "name":"Chn0007"
+            },
+            {
+               "hashMap":{
+                  "InterEnbOutAtt_X2HO":"250",
+                  "InterEnbOutSucc_X2HO":"170"
+               },
+               "name":"Chn0005"
+            }
+
+        ]
+  },
+  {
+    "additionalMeasurements": [
+    		{
+               "hashMap":{
+                  "networkId":"plmnid1",
+                  "InterEnbOutAtt_X2HO":"150",
+                  "InterEnbOutSucc_X2HO":"290"
+               },
+               "name":"Chn0007"
+            },
+            {
+               "hashMap":{
+                  "InterEnbOutAtt_X2HO":"250",
+                  "InterEnbOutSucc_X2HO":"170"
+               },
+               "name":"Chn0005"
+            }
+
+        ]
+  }
+  ]
+]
\ No newline at end of file
diff --git a/test/apps/pci_optimization/test_ml_model.py b/test/apps/pci_optimization/test_ml_model.py
new file mode 100644
index 0000000..9c617a9
--- /dev/null
+++ b/test/apps/pci_optimization/test_ml_model.py
@@ -0,0 +1,87 @@
+# -------------------------------------------------------------------------
+#   Copyright (C) 2020 Wipro Limited.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+import copy
+from mock import patch
+import unittest
+from apps.pci.optimizers.solver.ml_model import MlModel
+from osdf.adapters.dcae.des import DESException
+import osdf.config.loader as config_loader
+from osdf.utils.interfaces import json_from_file
+from osdf.utils.programming_utils import DotDict
+
+
+class TestMlModel(unittest.TestCase):
+    def setUp(self):
+        self.config_spec = {
+            "deployment": "config/osdf_config.yaml",
+            "core": "config/common_config.yaml"
+        }
+        self.osdf_config = DotDict(config_loader.all_configs(**self.config_spec))
+
+    def tearDown(self):
+        pass
+
+    def test_ml_model(self):
+        des_result_file = 'test/apps/pci_optimization/des_result.json'
+        results = json_from_file(des_result_file)
+
+        dzn_data = {
+            'NUM_NODES': 4,
+            'NUM_PCIS': 4,
+            'NUM_NEIGHBORS': 4,
+            'NEIGHBORS': [],
+            'NUM_SECOND_LEVEL_NEIGHBORS': 1,
+            'SECOND_LEVEL_NEIGHBORS': [],
+            'PCI_UNCHANGEABLE_CELLS': [],
+            'ORIGINAL_PCIS': []
+        }
+
+        network_cell_info = {
+            'cell_list': [
+                {
+                    'cell_id': 'Chn0001',
+                    'id': 1,
+                    'nbr_list': []
+                },
+                {
+                    'cell_id': 'Chn0002',
+                    'id': 2,
+                    'nbr_list': []
+                }
+            ]
+        }
+        self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', side_effect=results)
+        self.Mock_req = self.patcher_req.start()
+        mlmodel = MlModel()
+        mlmodel.get_additional_inputs(dzn_data, network_cell_info)
+        self.assertEqual(['Chn0001'], dzn_data['PCI_UNCHANGEABLE_CELLS'])
+        self.patcher_req.stop()
+
+        dzn_data['PCI_UNCHANGEABLE_CELLS'] = []
+        self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', side_effect=DESException('error'))
+        self.Mock_req = self.patcher_req.start()
+        mlmodel.get_additional_inputs(dzn_data, network_cell_info)
+        self.assertEqual([], dzn_data['PCI_UNCHANGEABLE_CELLS'])
+        self.patcher_req.stop()
+
+        self.patcher_req = patch('osdf.adapters.dcae.des.extract_data', return_value=[])
+        self.Mock_req = self.patcher_req.start()
+        mlmodel.get_additional_inputs(dzn_data, network_cell_info)
+        self.assertEqual([], dzn_data['PCI_UNCHANGEABLE_CELLS'])
+        self.patcher_req.stop()
diff --git a/test/config/common_config.yaml b/test/config/common_config.yaml
index a3ef82e..560d707 100644
--- a/test/config/common_config.yaml
+++ b/test/config/common_config.yaml
@@ -61,4 +61,14 @@ policy_info:
                     - get_param: subscriber_role
     default:  # if no explicit service related information is needed
         policy_fetch: by_name
-        policy_scope: none
\ No newline at end of file
+        policy_scope: none
+
+PCI:
+    ML:
+        average_ho_threshold: 10000
+        latest_ho_threshold: 500
+    DES:
+        service_id: ho_metric
+        filter:
+            interval: 10
+    ml_enabled: false
diff --git a/test/config/osdf_config.yaml b/test/config/osdf_config.yaml
index 2ed8b88..ef73d4c 100755
--- a/test/config/osdf_config.yaml
+++ b/test/config/osdf_config.yaml
@@ -70,3 +70,12 @@ osdfPCIOptPassword: PCI-OSDF-PASSWD
 
 aaiUrl: "https://aai.url:30233"
 aaiServiceInstanceUrl : "/aai/v20/nodes/service-instances/service-instance/"
+
+#DES api
+desUrl: https://des.url:9000
+desApiPath: /datalake/v1/exposure/
+desHeaders:
+  Accept: application/json
+  Content-Type: application/json
+desUsername:
+desPassword:
diff --git a/test/functest/simulators/simulated-config/common_config.yaml b/test/functest/simulators/simulated-config/common_config.yaml
index 1249dc0..36c639f 100644
--- a/test/functest/simulators/simulated-config/common_config.yaml
+++ b/test/functest/simulators/simulated-config/common_config.yaml
@@ -62,3 +62,13 @@ policy_info:
     default:  # if no explicit service related information is needed
         policy_fetch: by_name
         policy_scope: none
+
+PCI:
+    ML:
+        average_ho_threshold: 10000
+        latest_ho_threshold: 500
+    DES:
+        service_id: ho_metric
+        filter:
+            interval: 10
+    ml_enabled: false