Removed unused DB-adapters, test cases, 60+% cover

We are not using OracleDB, PostgresDB, and VerticaDB, so there is no
need to keep the "dead code" -- we can always add it back as needed.
Added test cases so that all the files are at least minimally covered.
Overall coverage on my local tox shows 66% coverage (I manually ensured
all files are included in the coverage report).

Issue-ID: OPTFRA-95
Change-Id: If1cab112236b4f32a96315308ce815088fa092d1
Signed-off-by: Sastry Isukapalli <sastry@research.att.com>

15 files changed, 184 insertions, 222 deletions

diff --git a/osdf/adapters/database/OracleDB.py b/osdf/adapters/database/OracleDB.py
deleted file mode 100644
index 655dd27..0000000
--- a/osdf/adapters/database/OracleDB.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# -------------------------------------------------------------------------
-#   Copyright (c) 2015-2017 AT&T Intellectual Property
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-# -------------------------------------------------------------------------
-#
-
-import cx_Oracle
-
-from osdf.utils.programming_utils import MetaSingleton
-
-
-class OracleDB(metaclass=MetaSingleton):
-    conn, cur = None, None
-
-    def connect(self, host=None, sid=None, user=None, passwd=None, port=5432):
-        if self.conn is None:
-            tns_info = cx_Oracle.makedsn(host=host, port=port, sid=sid)
-            self.conn = cx_Oracle.connect(user=user, password=passwd, dsn=tns_info, threaded=True)
-            self.cur = self.conn.cursor()
-        return self.conn, self.cur
diff --git a/osdf/adapters/database/PostgresDB.py b/osdf/adapters/database/PostgresDB.py
deleted file mode 100644
index 6689566..0000000
--- a/osdf/adapters/database/PostgresDB.py
+++ /dev/null
@@ -1,31 +0,0 @@
-# -------------------------------------------------------------------------
-#   Copyright (c) 2015-2017 AT&T Intellectual Property
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-# -------------------------------------------------------------------------
-#
-
-import psycopg2
-
-from osdf.utils.programming_utils import MetaSingleton
-
-
-class PostgresDB(metaclass=MetaSingleton):
-    conn, cur = None, None
-
-    def connect(self, host=None, db=None, user=None, passwd=None, port=5432):
-        if self.conn is None:
-            self.conn = psycopg2.connect(host=host, port=port, user=user, password=passwd, database=db)
-            self.cur = self.conn.cursor()
-        return self.conn, self.cur
diff --git a/osdf/adapters/database/VerticaDB.py b/osdf/adapters/database/VerticaDB.py
deleted file mode 100644
index ad961d7..0000000
--- a/osdf/adapters/database/VerticaDB.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# -------------------------------------------------------------------------
-#   Copyright (c) 2015-2017 AT&T Intellectual Property
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-# -------------------------------------------------------------------------
-#
-
-import jaydebeapi
-import sqlalchemy.pool as pool
-
-from jaydebeapi import _DEFAULT_CONVERTERS, _java_to_py
-from osdf.utils.programming_utils import MetaSingleton
-from osdf.config.base import osdf_config
-
-_DEFAULT_CONVERTERS.update({'BIGINT': _java_to_py('longValue')})
-
-
-class VerticaDB(metaclass=MetaSingleton):
-    connection_pool = None
-
-    def get_connection(self):
-        p = self.get_config_params()
-        c = jaydebeapi.connect(
-            'com.vertica.jdbc.Driver',
-            'jdbc:vertica://{}:{}/{}'.format(p['host'], p['port'], p['db']),
-            {'user': p['user'], 'password': p['passwd'], 'CHARSET': 'UTF8'},
-            jars=[p['db_driver']]
-        )
-        return c
-
-    def get_config_params(self):
-        config = osdf_config["deployment"]
-        host, port, db = config["verticaHost"], config["verticaPort"], config.get("verticaDB")
-        user, passwd = config["verticaUsername"], config["verticaPassword"]
-        jar_path = osdf_config['core']['osdf_system']['vertica_jar']
-        params = dict(host=host, db=db, user=user, passwd=passwd, port=port, db_driver=jar_path)
-        return params
-
-    def connect(self):
-        if self.connection_pool is None:
-            self.connection_pool = pool.QueuePool(self.get_connection, max_overflow=10, pool_size=5, recycle=600)
-        conn = self.connection_pool.connect()
-        cursor = conn.cursor()
-        return conn, cursor
diff --git a/osdf/adapters/database/__init__.py b/osdf/adapters/database/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/osdf/adapters/database/__init__.py
+++ /dev/null
diff --git a/osdf/adapters/request_parsing/__init__.py b/osdf/adapters/request_parsing/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/osdf/adapters/request_parsing/__init__.py
+++ /dev/null
diff --git a/osdf/adapters/request_parsing/placement.py b/osdf/adapters/request_parsing/placement.py
deleted file mode 100644
index d7a6575..0000000
--- a/osdf/adapters/request_parsing/placement.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# -------------------------------------------------------------------------
-#   Copyright (c) 2015-2017 AT&T Intellectual Property
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-# -------------------------------------------------------------------------
-#
-
-import copy
-import json
-from osdf.utils.programming_utils import list_flatten, dot_notation
-
-
-def json_path_after_expansion(req_json, reference):
-    """
-    Get the child node(s) from the dot-notation [reference] and parent [req_json].
-    For placement and other requests, there are encoded JSONs inside the request or policy,
-    so we need to expand it and then do a search over the parent plus expanded JSON.
-    """
-    req_json_copy = copy.deepcopy(req_json)  # since we expand the JSON in place, we work on a copy
-    req_json_copy['placementInfo']['orderInfo'] = json.loads(req_json_copy['placementInfo']['orderInfo'])
-    info = dot_notation(req_json_copy, reference)
-    return list_flatten(info) if isinstance(info, list) else info
diff --git a/osdf/adapters/sdc/asdc.py b/osdf/adapters/sdc/sdc.py
index 43932ba..43932ba 100755
--- a/osdf/adapters/sdc/asdc.py
+++ b/osdf/adapters/sdc/sdc.py
diff --git a/osdf/models/api/placementRequest.py b/osdf/models/api/placementRequest.py
index 73eac75..df5f931 100644
--- a/osdf/models/api/placementRequest.py
+++ b/osdf/models/api/placementRequest.py
@@ -17,8 +17,8 @@
 #
 
 from .common import OSDFModel
-from schematics.types import StringType, URLType, IntType, FloatType
-from schematics.types.compound import ModelType, ListType
+from schematics.types import BaseType, StringType, URLType, IntType
+from schematics.types.compound import ModelType, ListType, DictType
 
 
 class RequestInfo(OSDFModel):
@@ -27,7 +27,7 @@ class RequestInfo(OSDFModel):
     requestId = StringType(required=True)
     callbackUrl = URLType(required=True)
     sourceId = StringType(required=True)
-    optimizer = ListType(StringType())
+    optimizers = ListType(StringType(required=True))
     numSolutions = IntType()
     timeout = IntType()
     requestType = StringType()
@@ -47,7 +47,6 @@ class ResourceModelInfo(OSDFModel):
     modelVersion = StringType()
     modelVersionId = StringType()
     modelType = StringType()
-    operationalStatus = StringType()
 
 
 class ExistingLicenseInfo(OSDFModel):
@@ -70,10 +69,9 @@ class PlacementDemand(OSDFModel):
     exclusionCandidateInfo = ListType(ModelType(CandidateInfo))
     requiredCandidateInfo = ListType(ModelType(CandidateInfo))
     resourceModelInfo = ModelType(ResourceModelInfo)
-    tenantId = StringType()
+    tenantId = StringType(required=True)
     tenantName = StringType()
 
-
 class ExistingPlacementInfo(OSDFModel):
     serviceInstanceId = StringType(required=True)
 
@@ -100,22 +98,15 @@ class ServiceModelInfo(OSDFModel):
     modelVersion = StringType(required=True)
 
 
-class Location(OSDFModel):
-    latitude = FloatType(required=True)
-    longitude = FloatType(required=True)
-
-
 class PlacementInfo(OSDFModel):
     """Information specific to placement optimization"""
-    serviceModelInfo = ModelType(ServiceModelInfo)
-    subscriberInfo = ModelType(SubscriberInfo)
+    serviceModelInfo = ModelType(ServiceModelInfo, required=True)
+    subscriberInfo = ModelType(SubscriberInfo, required=True)
     demandInfo = ModelType(DemandInfo, required=True)
-    orderInfo = StringType()
+    requestParameters = DictType(BaseType)
     policyId = ListType(StringType())
-    serviceInstanceId = StringType()
+    serviceInstanceId = StringType(required=True)
     existingPlacement = ModelType(ExistingPlacementInfo)
-    location = ModelType(Location)
-    serviceType = StringType()
 
 
 class PlacementAPI(OSDFModel):
diff --git a/osdf/optimizers/licenseopt/simple_license_allocation.py b/osdf/optimizers/licenseopt/simple_license_allocation.py
index 1b5b670..beafbe4 100644
--- a/osdf/optimizers/licenseopt/simple_license_allocation.py
+++ b/osdf/optimizers/licenseopt/simple_license_allocation.py
@@ -19,10 +19,9 @@
 import json
 
 from requests import RequestException
-from osdf.datasources.sdc import sdc, constraint_handler
+from osdf.adapters.sdc import sdc, constraint_handler
 from osdf.logging.osdf_logging import audit_log, metrics_log, MH
 from osdf.config.base import osdf_config
-from osdf.utils import data_mapping
 
 
 def license_optim(request_json):
@@ -36,11 +35,12 @@ def license_optim(request_json):
     config = osdf_config.deployment
 
     model_name = request_json['placementInfo']['serviceModelInfo']['modelName']
-    service_name = data_mapping.get_service_type(model_name)
+    # service_name = data_mapping.get_service_type(model_name)
+    service_name = model_name
 
     license_info = []
 
-    order_info = json.loads(request_json["placementInfo"]["orderInfo"])
+    order_info = json.loads(request_json["placementInfo"]["requestParameters"])
     if service_name == 'VPE':
         data_mapping.normalize_user_params(order_info)
     for licenseDemand in request_json['placementInfo']['demandInfo']['licenseDemand']:
diff --git a/osdf/utils/local_processing.py b/osdf/utils/local_processing.py
deleted file mode 100644
index 6768839..0000000
--- a/osdf/utils/local_processing.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# -------------------------------------------------------------------------
-#   Copyright (c) 2015-2017 AT&T Intellectual Property
-#
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-#
-# -------------------------------------------------------------------------
-#
-
-import os
-
-from osdf.logging.osdf_logging import metrics_log, MH, warn_audit_error
-
-
-def local_create_job_file(req_id, json_req, fname='osdf-req-data.json'):
-    """Creates a "work" folder for local processing and place relevant
-    job task file in there"""
-
-    work_dir = 'osdf-optim/work/' + req_id
-    work_file = '{}/{}'.format(work_dir, fname)
-    try:
-        cur_task = "Making a local directory in the OSDF manager for req-id: {}".format(req_id)
-        metrics_log.info(MH.creating_local_env(cur_task))
-        os.makedirs(work_dir, exist_ok=True)
-    except Exception as err:
-        warn_audit_error(MH.error_local_env(req_id, "Can't create directory {}".format(work_dir), err))
-        return None
-    try:
-        with open(work_file, 'w') as fid:
-            fid.write(json_req['payload'])
-        return work_dir
-    except Exception as err:
-        warn_audit_error(MH.error_local_env(req_id, "can't create file {}".format(work_file), err))
-        return None
diff --git a/osdfapp.py b/osdfapp.py
index f854dca..3feb22b 100755
--- a/osdfapp.py
+++ b/osdfapp.py
@@ -31,14 +31,12 @@ import json
 import osdf.adapters.policy.interface
 import osdf.config.credentials
 import osdf.config.loader
-import osdf.datasources.aai.aai_local_cached_data
+# import osdf.datasources.aai.aai_local_cached_data
 import osdf.operation.error_handling
 import osdf.operation.responses
 import traceback
 from osdf.adapters.policy.interface import get_policies
-from osdf.adapters.response_parsing.aots_ueb_cm_data import aots_ds_ueb_listener
-from osdf.config.base import osdf_config, DOCKER_CM_OPTIMIZER, AOTS_CM_MESSAGE_BUS
-from osdf.optimizers.cmopt.rcscheduler.local_opt_processor import process_local_cm_scheduler_opt
+from osdf.config.base import osdf_config
 from osdf.optimizers.placementopt.conductor.remote_opt_processor import process_placement_opt
 from osdf.webapp.appcontroller import auth_basic
 from optparse import OptionParser
@@ -47,8 +45,7 @@ from osdf.operation.error_handling import request_exception_to_json_body, intern
 from requests import RequestException
 from schematics.exceptions import DataError
 from osdf.logging.osdf_logging import MH, audit_log, error_log
-from osdf.models.placementRequest import PlacementAPI
-from osdf.models.schedulerRequest import SchedulerAPI
+from osdf.models.api.placementRequest import PlacementAPI
 
 ERROR_TEMPLATE = osdf.ERROR_TEMPLATE
 
@@ -125,7 +122,7 @@ def do_placement_opt():
 
 # Returned when unexpected coding errors occur during initial synchronous processing
 @app.errorhandler(500)
-def interal_failure(error):
+def internal_failure(error):
     error_log.error("Synchronous error for request id {} {}".format(g.request_id, traceback.format_exc()))
     response = Response(internal_error_message, content_type='application/json; charset=utf-8')
     response.status_code = 500
diff --git a/test/adapters/test_message_router.py b/test/adapters/test_message_router.py
new file mode 100644
index 0000000..2a02dc8
--- /dev/null
+++ b/test/adapters/test_message_router.py
@@ -0,0 +1,44 @@
+import osdf.adapters.dcae.message_router as MR
+import unittest
+
+from osdf.operation.exceptions import MessageBusConfigurationException
+from unittest.mock import patch
+
+
+class TestMessageRouter(unittest.TestCase):
+
+    def test_valid_MR(self):
+        mr = MR.MessageRouterClient(dmaap_url="https://MYHOST:3905")
+
+    def test_valid_MR_with_base_urls(self):
+        base_urls = ["https://MYHOST1:3905/","https://MYHOST2:3905/"]
+        mr = MR.MessageRouterClient(mr_host_base_urls=base_urls, topic="MY-TOPIC")
+
+    def test_invalid_valid_MR_with_base_urls(self):
+        """Topic missing"""
+        base_urls = ["https://MYHOST1:3905/","https://MYHOST2:3905/"]
+        try:
+            mr = MR.MessageRouterClient(mr_host_base_urls=base_urls)
+        except MessageBusConfigurationException:
+            return
+
+        raise Exception("Allows invalid MR configuration") # if it failed to error out
+
+    @patch('osdf.adapters.dcae.message_router.MessageRouterClient.http_request', return_value={})
+    def test_mr_http_request_mocked(self, http_request):
+        mr = MR.MessageRouterClient(dmaap_url="https://MYHOST:3905")
+        mr.http_request = http_request
+        assert mr.get() == {} 
+        assert mr.post("Hello") == {} 
+
+    def test_mr_http_request_non_existent_host(self):
+        mr = MR.MessageRouterClient(dmaap_url="https://MYHOST:3905")
+        try:
+            mr.get()
+        except:
+            return
+
+        raise Exception("Allows invalid host") # if it failed to error out
+if __name__ == "__main__":
+    unittest.main()
+
diff --git a/test/mainapp/test_osdfapp.py b/test/mainapp/test_osdfapp.py
new file mode 100644
index 0000000..2ffe4f3
--- /dev/null
+++ b/test/mainapp/test_osdfapp.py
@@ -0,0 +1,57 @@
+import osdfapp
+import unittest
+
+from osdf.operation.exceptions import BusinessException
+from requests import Request, RequestException
+from schematics.exceptions import DataError
+from unittest import mock, TestCase
+from unittest.mock import patch
+
+
+class TestOSDFApp(TestCase):
+
+    def setUp(self):
+        self.patcher_g = patch('osdfapp.g', return_value={'request_id':'DUMMY-REQ'})
+        self.Mock_g = self.patcher_g.start()
+        # self.patcher2 = patch('package.module.Class2')
+        # self.MockClass2 = self.patcher2.start()
+
+    def tearDown(self):
+        patch.stopall()
+
+    def dummy_request_exception(self):
+        e = RequestException("Web Request Exception Description")
+        e.response = mock.MagicMock()
+        e.request = Request(method="GET", url="SOME-URL")
+        e.response.status_code = 400
+        e.response.content = "Some request exception occurred"
+        # request().raise_for_status.side_effect = e
+        return e
+ 
+    def test_handle_business_exception(self):
+        e = BusinessException("Business Exception Description")
+        resp = osdfapp.handle_business_exception(e)
+        assert resp.status_code == 400
+
+    def test_handle_request_exception(self):
+        e = self.dummy_request_exception()
+        resp = osdfapp.handle_request_exception(e)
+        assert resp.status_code == 400
+
+    def test_handle_data_error(self):
+        e = DataError({"A1": "A1 Data Error"})
+        resp = osdfapp.handle_data_error(e)
+        assert resp.status_code == 400
+
+    def test_internal_failure(self):
+        e = Exception("An Internal Error")
+        resp = osdfapp.internal_failure(e)
+        assert resp.status_code == 500
+
+    def test_getOptions_default(self):
+        opts = osdfapp.getOptions(["PROG"])  # ensure nothing breaks
+
+
+if __name__ == "__main__":
+    unittest.main()
+
diff --git a/test/test_api_validation.py b/test/test_api_validation.py
new file mode 100644
index 0000000..5af81e6
--- /dev/null
+++ b/test/test_api_validation.py
@@ -0,0 +1,29 @@
+import json
+import unittest
+
+from osdf.models.api.placementRequest import PlacementAPI
+from osdf.models.api.placementResponse import PlacementResponse
+from schematics.exceptions import ModelValidationError
+
+
+class TestReqValidation(unittest.TestCase):
+
+    def test_req_validation(self):
+        req_file = "./test/placement-tests/request.json"
+        req_json = json.loads(open(req_file).read())
+        self.assertEqual(PlacementAPI(req_json).validate(), None)
+
+    def test_req_failure(self):
+        req_json = {}
+        self.assertRaises(ModelValidationError, lambda: PlacementAPI(req_json).validate())
+
+
+class TestResponseValidation(unittest.TestCase):
+
+    def test_invalid_response(self):
+        resp_json = {}
+        self.assertRaises(ModelValidationError, lambda: PlacementResponse(resp_json).validate())
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/test_process_placement_opt.py b/test/test_process_placement_opt.py
new file mode 100644
index 0000000..5d3014b
--- /dev/null
+++ b/test/test_process_placement_opt.py
@@ -0,0 +1,38 @@
+import unittest
+import json
+import yaml
+from osdf.optimizers.placementopt.conductor.remote_opt_processor import process_placement_opt
+from mock import patch
+
+class TestConductorApiBuilder(unittest.TestCase):
+
+    def test_conductor_api_call_builder(self):
+        #main_dir = ".."
+        main_dir = ""
+        conductor_api_template = main_dir + "osdf/templates/conductor_interface.json"
+        parameter_data_file = main_dir + "test/placement-tests/request.json"
+        policy_data_path = main_dir + "test/policy-local-files/"
+        local_config_file = main_dir + "config/common_config.yaml"
+
+        policy_data_files = ["CloudAttributePolicy_vGMuxInfra_1.json",
+                            "CloudAttributePolicy_vG_1.json",
+                            "DistanceToLocationPolicy_vGMuxInfra_1.json",
+                            "DistanceToLocationPolicy_vG_1.json",
+                            "InventoryGroup_vGMuxInfra_1.json",
+                            "InventoryGroup_vG_1.json",
+                            "PlacementOptimizationPolicy.json",
+                            "ResourceInstancePolicy_vG_1.json",
+                            "VNFPolicy_vGMuxInfra_1.json",
+                            "VNFPolicy_vG_1.json",
+                            "ZonePolicy_vGMuxInfra_1.json",
+                            "ZonePolicy_vG_1.json"]
+        request_json = json.loads(open(parameter_data_file).read())
+        policies = [json.loads(open(policy_data_path + file).read()) for file in policy_data_files]
+        local_config = yaml.load(open(local_config_file))
+        with patch('osdf.optimizers.placementopt.conductor.conductor.request', return_value={"solutionInfo": {"placementInfo": "dummy"}}):
+            templ_string = process_placement_opt(request_json, policies, local_config, [])
+
+
+if __name__ == "__main__":
+    unittest.main()
+