diff options
| -rw-r--r-- | osdf/adapters/database/OracleDB.py | 32 | ||||
| -rw-r--r-- | osdf/adapters/database/PostgresDB.py | 31 | ||||
| -rw-r--r-- | osdf/adapters/database/VerticaDB.py | 55 | ||||
| -rw-r--r-- | osdf/adapters/database/__init__.py | 0 | ||||
| -rw-r--r-- | osdf/adapters/request_parsing/__init__.py | 0 | ||||
| -rw-r--r-- | osdf/adapters/request_parsing/placement.py | 33 | ||||
| -rwxr-xr-x | osdf/adapters/sdc/sdc.py (renamed from osdf/adapters/sdc/asdc.py) | 0 | ||||
| -rw-r--r-- | osdf/models/api/placementRequest.py | 25 | ||||
| -rw-r--r-- | osdf/optimizers/licenseopt/simple_license_allocation.py | 8 | ||||
| -rw-r--r-- | osdf/utils/local_processing.py | 43 | ||||
| -rwxr-xr-x | osdfapp.py | 11 | ||||
| -rw-r--r-- | test/adapters/test_message_router.py | 44 | ||||
| -rw-r--r-- | test/mainapp/test_osdfapp.py | 57 | ||||
| -rw-r--r-- | test/test_api_validation.py | 29 | ||||
| -rw-r--r-- | test/test_process_placement_opt.py | 38 |
15 files changed, 184 insertions, 222 deletions
diff --git a/osdf/adapters/database/OracleDB.py b/osdf/adapters/database/OracleDB.py deleted file mode 100644 index 655dd27..0000000 --- a/osdf/adapters/database/OracleDB.py +++ /dev/null @@ -1,32 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) 2015-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -import cx_Oracle - -from osdf.utils.programming_utils import MetaSingleton - - -class OracleDB(metaclass=MetaSingleton): - conn, cur = None, None - - def connect(self, host=None, sid=None, user=None, passwd=None, port=5432): - if self.conn is None: - tns_info = cx_Oracle.makedsn(host=host, port=port, sid=sid) - self.conn = cx_Oracle.connect(user=user, password=passwd, dsn=tns_info, threaded=True) - self.cur = self.conn.cursor() - return self.conn, self.cur diff --git a/osdf/adapters/database/PostgresDB.py b/osdf/adapters/database/PostgresDB.py deleted file mode 100644 index 6689566..0000000 --- a/osdf/adapters/database/PostgresDB.py +++ /dev/null @@ -1,31 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) 2015-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -import psycopg2 - -from osdf.utils.programming_utils import MetaSingleton - - -class PostgresDB(metaclass=MetaSingleton): - conn, cur = None, None - - def connect(self, host=None, db=None, user=None, passwd=None, port=5432): - if self.conn is None: - self.conn = psycopg2.connect(host=host, port=port, user=user, password=passwd, database=db) - self.cur = self.conn.cursor() - return self.conn, self.cur diff --git a/osdf/adapters/database/VerticaDB.py b/osdf/adapters/database/VerticaDB.py deleted file mode 100644 index ad961d7..0000000 --- a/osdf/adapters/database/VerticaDB.py +++ /dev/null @@ -1,55 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) 2015-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -import jaydebeapi -import sqlalchemy.pool as pool - -from jaydebeapi import _DEFAULT_CONVERTERS, _java_to_py -from osdf.utils.programming_utils import MetaSingleton -from osdf.config.base import osdf_config - -_DEFAULT_CONVERTERS.update({'BIGINT': _java_to_py('longValue')}) - - -class VerticaDB(metaclass=MetaSingleton): - connection_pool = None - - def get_connection(self): - p = self.get_config_params() - c = jaydebeapi.connect( - 'com.vertica.jdbc.Driver', - 'jdbc:vertica://{}:{}/{}'.format(p['host'], p['port'], p['db']), - {'user': p['user'], 'password': p['passwd'], 'CHARSET': 'UTF8'}, - jars=[p['db_driver']] - ) - return c - - def get_config_params(self): - config = osdf_config["deployment"] - host, port, db = config["verticaHost"], config["verticaPort"], config.get("verticaDB") - user, passwd = config["verticaUsername"], config["verticaPassword"] - jar_path = osdf_config['core']['osdf_system']['vertica_jar'] - params = dict(host=host, db=db, user=user, passwd=passwd, port=port, db_driver=jar_path) - return params - - def connect(self): - if self.connection_pool is None: - self.connection_pool = pool.QueuePool(self.get_connection, max_overflow=10, pool_size=5, recycle=600) - conn = self.connection_pool.connect() - cursor = conn.cursor() - return conn, cursor diff --git a/osdf/adapters/database/__init__.py b/osdf/adapters/database/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/osdf/adapters/database/__init__.py +++ /dev/null diff --git a/osdf/adapters/request_parsing/__init__.py b/osdf/adapters/request_parsing/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/osdf/adapters/request_parsing/__init__.py +++ /dev/null diff --git a/osdf/adapters/request_parsing/placement.py b/osdf/adapters/request_parsing/placement.py deleted file mode 100644 index d7a6575..0000000 --- a/osdf/adapters/request_parsing/placement.py +++ /dev/null @@ -1,33 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) 2015-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -import copy -import json -from osdf.utils.programming_utils import list_flatten, dot_notation - - -def json_path_after_expansion(req_json, reference): - """ - Get the child node(s) from the dot-notation [reference] and parent [req_json]. - For placement and other requests, there are encoded JSONs inside the request or policy, - so we need to expand it and then do a search over the parent plus expanded JSON. - """ - req_json_copy = copy.deepcopy(req_json) # since we expand the JSON in place, we work on a copy - req_json_copy['placementInfo']['orderInfo'] = json.loads(req_json_copy['placementInfo']['orderInfo']) - info = dot_notation(req_json_copy, reference) - return list_flatten(info) if isinstance(info, list) else info diff --git a/osdf/adapters/sdc/asdc.py b/osdf/adapters/sdc/sdc.py index 43932ba..43932ba 100755 --- a/osdf/adapters/sdc/asdc.py +++ b/osdf/adapters/sdc/sdc.py diff --git a/osdf/models/api/placementRequest.py b/osdf/models/api/placementRequest.py index 73eac75..df5f931 100644 --- a/osdf/models/api/placementRequest.py +++ b/osdf/models/api/placementRequest.py @@ -17,8 +17,8 @@ # from .common import OSDFModel -from schematics.types import StringType, URLType, IntType, FloatType -from schematics.types.compound import ModelType, ListType +from schematics.types import BaseType, StringType, URLType, IntType +from schematics.types.compound import ModelType, ListType, DictType class RequestInfo(OSDFModel): @@ -27,7 +27,7 @@ class RequestInfo(OSDFModel): requestId = StringType(required=True) callbackUrl = URLType(required=True) sourceId = StringType(required=True) - optimizer = ListType(StringType()) + optimizers = ListType(StringType(required=True)) numSolutions = IntType() timeout = IntType() requestType = StringType() @@ -47,7 +47,6 @@ class ResourceModelInfo(OSDFModel): modelVersion = StringType() modelVersionId = StringType() modelType = StringType() - operationalStatus = StringType() class ExistingLicenseInfo(OSDFModel): @@ -70,10 +69,9 @@ class PlacementDemand(OSDFModel): exclusionCandidateInfo = ListType(ModelType(CandidateInfo)) requiredCandidateInfo = ListType(ModelType(CandidateInfo)) resourceModelInfo = ModelType(ResourceModelInfo) - tenantId = StringType() + tenantId = StringType(required=True) tenantName = StringType() - class ExistingPlacementInfo(OSDFModel): serviceInstanceId = StringType(required=True) @@ -100,22 +98,15 @@ class ServiceModelInfo(OSDFModel): modelVersion = StringType(required=True) -class Location(OSDFModel): - latitude = FloatType(required=True) - longitude = FloatType(required=True) - - class PlacementInfo(OSDFModel): """Information specific to placement optimization""" - serviceModelInfo = ModelType(ServiceModelInfo) - subscriberInfo = ModelType(SubscriberInfo) + serviceModelInfo = ModelType(ServiceModelInfo, required=True) + subscriberInfo = ModelType(SubscriberInfo, required=True) demandInfo = ModelType(DemandInfo, required=True) - orderInfo = StringType() + requestParameters = DictType(BaseType) policyId = ListType(StringType()) - serviceInstanceId = StringType() + serviceInstanceId = StringType(required=True) existingPlacement = ModelType(ExistingPlacementInfo) - location = ModelType(Location) - serviceType = StringType() class PlacementAPI(OSDFModel): diff --git a/osdf/optimizers/licenseopt/simple_license_allocation.py b/osdf/optimizers/licenseopt/simple_license_allocation.py index 1b5b670..beafbe4 100644 --- a/osdf/optimizers/licenseopt/simple_license_allocation.py +++ b/osdf/optimizers/licenseopt/simple_license_allocation.py @@ -19,10 +19,9 @@ import json from requests import RequestException -from osdf.datasources.sdc import sdc, constraint_handler +from osdf.adapters.sdc import sdc, constraint_handler from osdf.logging.osdf_logging import audit_log, metrics_log, MH from osdf.config.base import osdf_config -from osdf.utils import data_mapping def license_optim(request_json): @@ -36,11 +35,12 @@ def license_optim(request_json): config = osdf_config.deployment model_name = request_json['placementInfo']['serviceModelInfo']['modelName'] - service_name = data_mapping.get_service_type(model_name) + # service_name = data_mapping.get_service_type(model_name) + service_name = model_name license_info = [] - order_info = json.loads(request_json["placementInfo"]["orderInfo"]) + order_info = json.loads(request_json["placementInfo"]["requestParameters"]) if service_name == 'VPE': data_mapping.normalize_user_params(order_info) for licenseDemand in request_json['placementInfo']['demandInfo']['licenseDemand']: diff --git a/osdf/utils/local_processing.py b/osdf/utils/local_processing.py deleted file mode 100644 index 6768839..0000000 --- a/osdf/utils/local_processing.py +++ /dev/null @@ -1,43 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) 2015-2017 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -import os - -from osdf.logging.osdf_logging import metrics_log, MH, warn_audit_error - - -def local_create_job_file(req_id, json_req, fname='osdf-req-data.json'): - """Creates a "work" folder for local processing and place relevant - job task file in there""" - - work_dir = 'osdf-optim/work/' + req_id - work_file = '{}/{}'.format(work_dir, fname) - try: - cur_task = "Making a local directory in the OSDF manager for req-id: {}".format(req_id) - metrics_log.info(MH.creating_local_env(cur_task)) - os.makedirs(work_dir, exist_ok=True) - except Exception as err: - warn_audit_error(MH.error_local_env(req_id, "Can't create directory {}".format(work_dir), err)) - return None - try: - with open(work_file, 'w') as fid: - fid.write(json_req['payload']) - return work_dir - except Exception as err: - warn_audit_error(MH.error_local_env(req_id, "can't create file {}".format(work_file), err)) - return None @@ -31,14 +31,12 @@ import json import osdf.adapters.policy.interface import osdf.config.credentials import osdf.config.loader -import osdf.datasources.aai.aai_local_cached_data +# import osdf.datasources.aai.aai_local_cached_data import osdf.operation.error_handling import osdf.operation.responses import traceback from osdf.adapters.policy.interface import get_policies -from osdf.adapters.response_parsing.aots_ueb_cm_data import aots_ds_ueb_listener -from osdf.config.base import osdf_config, DOCKER_CM_OPTIMIZER, AOTS_CM_MESSAGE_BUS -from osdf.optimizers.cmopt.rcscheduler.local_opt_processor import process_local_cm_scheduler_opt +from osdf.config.base import osdf_config from osdf.optimizers.placementopt.conductor.remote_opt_processor import process_placement_opt from osdf.webapp.appcontroller import auth_basic from optparse import OptionParser @@ -47,8 +45,7 @@ from osdf.operation.error_handling import request_exception_to_json_body, intern from requests import RequestException from schematics.exceptions import DataError from osdf.logging.osdf_logging import MH, audit_log, error_log -from osdf.models.placementRequest import PlacementAPI -from osdf.models.schedulerRequest import SchedulerAPI +from osdf.models.api.placementRequest import PlacementAPI ERROR_TEMPLATE = osdf.ERROR_TEMPLATE @@ -125,7 +122,7 @@ def do_placement_opt(): # Returned when unexpected coding errors occur during initial synchronous processing @app.errorhandler(500) -def interal_failure(error): +def internal_failure(error): error_log.error("Synchronous error for request id {} {}".format(g.request_id, traceback.format_exc())) response = Response(internal_error_message, content_type='application/json; charset=utf-8') response.status_code = 500 diff --git a/test/adapters/test_message_router.py b/test/adapters/test_message_router.py new file mode 100644 index 0000000..2a02dc8 --- /dev/null +++ b/test/adapters/test_message_router.py @@ -0,0 +1,44 @@ +import osdf.adapters.dcae.message_router as MR +import unittest + +from osdf.operation.exceptions import MessageBusConfigurationException +from unittest.mock import patch + + +class TestMessageRouter(unittest.TestCase): + + def test_valid_MR(self): + mr = MR.MessageRouterClient(dmaap_url="https://MYHOST:3905") + + def test_valid_MR_with_base_urls(self): + base_urls = ["https://MYHOST1:3905/","https://MYHOST2:3905/"] + mr = MR.MessageRouterClient(mr_host_base_urls=base_urls, topic="MY-TOPIC") + + def test_invalid_valid_MR_with_base_urls(self): + """Topic missing""" + base_urls = ["https://MYHOST1:3905/","https://MYHOST2:3905/"] + try: + mr = MR.MessageRouterClient(mr_host_base_urls=base_urls) + except MessageBusConfigurationException: + return + + raise Exception("Allows invalid MR configuration") # if it failed to error out + + @patch('osdf.adapters.dcae.message_router.MessageRouterClient.http_request', return_value={}) + def test_mr_http_request_mocked(self, http_request): + mr = MR.MessageRouterClient(dmaap_url="https://MYHOST:3905") + mr.http_request = http_request + assert mr.get() == {} + assert mr.post("Hello") == {} + + def test_mr_http_request_non_existent_host(self): + mr = MR.MessageRouterClient(dmaap_url="https://MYHOST:3905") + try: + mr.get() + except: + return + + raise Exception("Allows invalid host") # if it failed to error out +if __name__ == "__main__": + unittest.main() + diff --git a/test/mainapp/test_osdfapp.py b/test/mainapp/test_osdfapp.py new file mode 100644 index 0000000..2ffe4f3 --- /dev/null +++ b/test/mainapp/test_osdfapp.py @@ -0,0 +1,57 @@ +import osdfapp +import unittest + +from osdf.operation.exceptions import BusinessException +from requests import Request, RequestException +from schematics.exceptions import DataError +from unittest import mock, TestCase +from unittest.mock import patch + + +class TestOSDFApp(TestCase): + + def setUp(self): + self.patcher_g = patch('osdfapp.g', return_value={'request_id':'DUMMY-REQ'}) + self.Mock_g = self.patcher_g.start() + # self.patcher2 = patch('package.module.Class2') + # self.MockClass2 = self.patcher2.start() + + def tearDown(self): + patch.stopall() + + def dummy_request_exception(self): + e = RequestException("Web Request Exception Description") + e.response = mock.MagicMock() + e.request = Request(method="GET", url="SOME-URL") + e.response.status_code = 400 + e.response.content = "Some request exception occurred" + # request().raise_for_status.side_effect = e + return e + + def test_handle_business_exception(self): + e = BusinessException("Business Exception Description") + resp = osdfapp.handle_business_exception(e) + assert resp.status_code == 400 + + def test_handle_request_exception(self): + e = self.dummy_request_exception() + resp = osdfapp.handle_request_exception(e) + assert resp.status_code == 400 + + def test_handle_data_error(self): + e = DataError({"A1": "A1 Data Error"}) + resp = osdfapp.handle_data_error(e) + assert resp.status_code == 400 + + def test_internal_failure(self): + e = Exception("An Internal Error") + resp = osdfapp.internal_failure(e) + assert resp.status_code == 500 + + def test_getOptions_default(self): + opts = osdfapp.getOptions(["PROG"]) # ensure nothing breaks + + +if __name__ == "__main__": + unittest.main() + diff --git a/test/test_api_validation.py b/test/test_api_validation.py new file mode 100644 index 0000000..5af81e6 --- /dev/null +++ b/test/test_api_validation.py @@ -0,0 +1,29 @@ +import json +import unittest + +from osdf.models.api.placementRequest import PlacementAPI +from osdf.models.api.placementResponse import PlacementResponse +from schematics.exceptions import ModelValidationError + + +class TestReqValidation(unittest.TestCase): + + def test_req_validation(self): + req_file = "./test/placement-tests/request.json" + req_json = json.loads(open(req_file).read()) + self.assertEqual(PlacementAPI(req_json).validate(), None) + + def test_req_failure(self): + req_json = {} + self.assertRaises(ModelValidationError, lambda: PlacementAPI(req_json).validate()) + + +class TestResponseValidation(unittest.TestCase): + + def test_invalid_response(self): + resp_json = {} + self.assertRaises(ModelValidationError, lambda: PlacementResponse(resp_json).validate()) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_process_placement_opt.py b/test/test_process_placement_opt.py new file mode 100644 index 0000000..5d3014b --- /dev/null +++ b/test/test_process_placement_opt.py @@ -0,0 +1,38 @@ +import unittest +import json +import yaml +from osdf.optimizers.placementopt.conductor.remote_opt_processor import process_placement_opt +from mock import patch + +class TestConductorApiBuilder(unittest.TestCase): + + def test_conductor_api_call_builder(self): + #main_dir = ".." + main_dir = "" + conductor_api_template = main_dir + "osdf/templates/conductor_interface.json" + parameter_data_file = main_dir + "test/placement-tests/request.json" + policy_data_path = main_dir + "test/policy-local-files/" + local_config_file = main_dir + "config/common_config.yaml" + + policy_data_files = ["CloudAttributePolicy_vGMuxInfra_1.json", + "CloudAttributePolicy_vG_1.json", + "DistanceToLocationPolicy_vGMuxInfra_1.json", + "DistanceToLocationPolicy_vG_1.json", + "InventoryGroup_vGMuxInfra_1.json", + "InventoryGroup_vG_1.json", + "PlacementOptimizationPolicy.json", + "ResourceInstancePolicy_vG_1.json", + "VNFPolicy_vGMuxInfra_1.json", + "VNFPolicy_vG_1.json", + "ZonePolicy_vGMuxInfra_1.json", + "ZonePolicy_vG_1.json"] + request_json = json.loads(open(parameter_data_file).read()) + policies = [json.loads(open(policy_data_path + file).read()) for file in policy_data_files] + local_config = yaml.load(open(local_config_file)) + with patch('osdf.optimizers.placementopt.conductor.conductor.request', return_value={"solutionInfo": {"placementInfo": "dummy"}}): + templ_string = process_placement_opt(request_json, policies, local_config, []) + + +if __name__ == "__main__": + unittest.main() + |