diff options
70 files changed, 2291 insertions, 331 deletions
diff --git a/.coveragerc b/.coveragerc index a5afd52..1fa0d3b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,7 @@ [run] branch = True cover_pylib = False -include = osdf/**/*.py +include = osdf/**/*.py, apps/**/*.py, runtime/*.py, runtime/**/*.py [report] # Regexes for lines to exclude from consideration @@ -29,6 +29,9 @@ wheels/ .installed.cfg *.egg MANIFEST +AUTHORS +ChangeLog +logs/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..3797dc8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,20 @@ +--- +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Required +version: 2 + +formats: + - htmlzip + +build: + image: latest + +python: + version: 3.7 + install: + - requirements: docs/requirements-docs.txt + +sphinx: + configuration: docs/conf.py @@ -3,78 +3,90 @@ project: 'optf-osdf' project_creation_date: '2017-07-06' lifecycle_state: 'Incubation' project_lead: &onap_releng_ptl - name: 'Shankaranarayanan Puzhavakath Narayanan' - email: 'snarayanan@research.att.com' - id: 'snarayanan' - company: 'ATT' - timezone: 'America/Bedminster' + name: 'Shankaranarayanan Puzhavakath Narayanan' + email: 'snarayanan@research.att.com' + id: 'snarayanan' + company: 'ATT' + timezone: 'America/Bedminster' project_category: '' primary_contact: *onap_releng_ptl issue_tracking: - type: 'jira' - url: 'https://jira.onap.org/projects/OPTFRA' - key: 'OPTFRA' + type: 'jira' + url: 'https://jira.onap.org/projects/OPTFRA' + key: 'OPTFRA' mailing_list: - type: 'groups.io' - url: 'lists.onap.org' - tag: '<[sub-project_name]>' + type: 'groups.io' + url: 'lists.onap.org' + tag: '<[sub-project_name]>' realtime_discussion: '' meetings: - - type: 'zoom' - agenda: 'https://wiki.onap.org/display/DW/Project+Resources+for+OOF' - url: 'https://wiki.onap.org/display/DW/Optimization+Framework+Project' - server: 'n/a' - channel: 'n/a' - repeats: 'weekly' - time: '15:00 UTC' + - type: 'zoom' + agenda: 'https://wiki.onap.org/display/DW/Project+Resources+for+OOF' + url: 'https://wiki.onap.org/display/DW/Optimization+Framework+Project' + server: 'n/a' + channel: 'n/a' + repeats: 'weekly' + time: '15:00 UTC' repositories: - - 'optf-cmso' - - 'optf-fgps' - - 'optf-has' - - 'optf-osdf' + - 'optf/osdf' committers: - - <<: *onap_releng_ptl - - name: 'Sarat Puthenpura' - email: 'sarat@research.att.com' - company: 'ATT' - id: 'sarat' - timezone: 'America/Bedminster' - - name: 'ramki krishnan' - email: 'ramkri123@gmail.com' - company: 'VMWare' - id: 'ramkri123' - timezone: 'America/Los_Angeles' - - name: 'Dileep Ranganathan' - email: 'dileep.ranganathan@intel.com' - company: 'Intel' - id: 'dileep.ranganathan' - timezone: 'America/Los_Angeles' - - name: 'Vikas Varma' - email: 'vikas.varma@att.com' - company: 'ATT' - id: 'vrvarma' - timezone: 'America/New_York' + - <<: *onap_releng_ptl + - name: 'Sarat Puthenpura' + email: 'sarat@research.att.com' + company: 'ATT' + id: 'sarat' + timezone: 'America/Bedminster' + - name: 'ramki krishnan' + email: 'ramkri123@gmail.com' + company: 'VMWare' + id: 'ramkri123' + timezone: 'America/Los_Angeles' + - name: 'Dileep Ranganathan' + email: 'dileep.ranganathan@intel.com' + company: 'Intel' + id: 'dileep.ranganathan' + timezone: 'America/Los_Angeles' + - name: 'Vikas Varma' + email: 'vikas.varma@att.com' + company: 'ATT' + id: 'vrvarma' + timezone: 'America/New_York' + - name: 'Arthur Martella' + email: 'arthur.martella.1@att.com' + company: 'ATT' + id: 'amartell' + timezone: 'America/New_York' tsc: - approval: 'https://lists.onap.org/pipermail/onap-tsc' - changes: - - type: 'Addition' - name: 'Ankitkumar Patel' - link: 'https://lists.onap.org/pipermail/onap-tsc/2018-April/004657.html' - - type: 'Removal' - name: 'maopeng zhang' - name: 'Sastry Isukapalli' - name: 'Yoram Zini' - link: 'https://lists.onap.org/pipermail/onap-tsc/2018-June/004975.html' - - type: 'Addition' - name: 'ramki krishnan' - name: 'Dileep Ranganathan' - link: 'https://lists.onap.org/g/ONAP-TSC/message/3205' - - type: 'Removal' - name: 'Ankitkumar Patel' - link: 'https://lists.onap.org/g/ONAP-TSC/message/3550' - - type: 'Addition' - name: 'Vikas Varma' - link: 'http://ircbot.wl.linuxfoundation.org/meetings/onap-meeting/2018/onap-meeting.2018-08-30-13.57.log.txt' - - type: 'Addition' - name: 'Shankaranarayanan Puzhavakath Narayanan' - link: 'https://civs.cs.cornell.edu/cgi-bin/results.pl?id=E_2696d1c15c2fdd16'
\ No newline at end of file + approval: 'https://lists.onap.org/pipermail/onap-tsc' + changes: + - type: 'Addition' + name: 'Ankitkumar Patel' + link: 'https://lists.onap.org/pipermail/onap-tsc/2018-April/004657.html' + - type: 'Removal' + name: 'maopeng zhang' + link: 'https://lists.onap.org/pipermail/onap-tsc/2018-June/004975.html' + - type: 'Removal' + name: 'Sastry Isukapalli' + link: 'https://lists.onap.org/pipermail/onap-tsc/2018-June/004975.html' + - type: 'Removal' + name: 'Yoram Zini' + link: 'https://lists.onap.org/pipermail/onap-tsc/2018-June/004975.html' + - type: 'Addition' + name: 'ramki krishnan' + link: 'https://lists.onap.org/g/ONAP-TSC/message/3205' + - type: 'Addition' + name: 'Dileep Ranganathan' + link: 'https://lists.onap.org/g/ONAP-TSC/message/3205' + - type: 'Removal' + name: 'Ankitkumar Patel' + link: 'https://lists.onap.org/g/ONAP-TSC/message/3550' + - type: 'Addition' + name: 'Vikas Varma' + link: 'https://wiki.onap.org/x/IplFAg' + - type: 'Addition' + name: 'Shankaranarayanan Puzhavakath Narayanan' + # yamllint disable-line rule:line-length + link: 'https://civs.cs.cornell.edu/cgi-bin/results.pl?id=E_2696d1c15c2fdd16' + - type: 'Addition' + name: 'Arthur Martella' + link: 'https://wiki.onap.org/x/qiVIB' diff --git a/README.md b/README.md new file mode 100644 index 0000000..0e2641a --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + + +#osdf + + +#cipher-utility + + python3 setup.py install + export PYTHONPATH=$PYTHONPATH:`pwd` + diff --git a/apps/route/optimizers/route_opt.mzn b/apps/route/optimizers/route_opt.mzn new file mode 100644 index 0000000..7aa73cb --- /dev/null +++ b/apps/route/optimizers/route_opt.mzn @@ -0,0 +1,53 @@ + +% Number of nodes +int: N; + % Start node +0..N-1: Start; + % End node +0..N-1: End; + % Number of edges (directed arcs) +int: M; + % The actual edges +set of int: Edges = 1..M; + % Edge lengths +array[Edges] of int: L; + % Edge start node +array[Edges] of 0..N-1: Edge_Start; +array[Edges] of 0..N-1: Edge_End; + + % Variable indicating if edge is used +array[Edges] of var 0..1: x; + +constraint + forall( i in 0..N-1 ) ( + if i = Start then + % outgoing flow + sum(e in Edges where Edge_Start[e] = i)(x[e]) - + % incoming flow + sum(e in Edges where Edge_End[e] = i)(x[e]) + = 1 + elseif i = End then + sum(e in Edges where Edge_Start[e] = i)(x[e]) - + sum(e in Edges where Edge_End[e] = i)(x[e]) + = -1 + else + sum(e in Edges where Edge_Start[e] = i)(x[e]) - + sum(e in Edges where Edge_End[e] = i)(x[e]) + = 0 + endif + ); + + +solve minimize sum(e in Edges)( L[e] * x[e] ); +%solve satisfy; + +output ["Length: ", show(sum(e in Edges)(L[e] * x[e])), "\n"] ++ + ["Start : ", show(Start), "\n"] ++ + ["End : ", show(End), "\n\n"] ++ + ["Edges in shortest path:\n"] ++ + [ if fix(x[e]) = 1 + then show(Edge_Start[e]) ++ " -> " ++ show(Edge_End[e]) ++ "\n" + else "" + endif | e in Edges + ]; + diff --git a/apps/route/optimizers/simple_route_opt.py b/apps/route/optimizers/simple_route_opt.py index b00180d..27c1141 100644 --- a/apps/route/optimizers/simple_route_opt.py +++ b/apps/route/optimizers/simple_route_opt.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------- -# Copyright (c) 2018 Huawei Intellectual Property +# Copyright (c) 2020 Huawei Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,12 @@ import requests from requests.auth import HTTPBasicAuth from osdf.utils.mdc_utils import mdc_from_json +from osdf.logging.osdf_logging import MH, audit_log, error_log, debug_log +import pymzn +from sklearn import preprocessing +import os +BASE_DIR = os.path.dirname(__file__) class RouteOpt: @@ -31,6 +36,8 @@ class RouteOpt: # DNS server and standard port of AAI.. # TODO: read the port from the configuration and add to DNS aai_host = "https://aai.api.simpledemo.onap.org:8443" + audit_log.info("base directory") + audit_log.info(BASE_DIR) aai_headers = { "X-TransactionId": "9999", "X-FromAppId": "OOF", @@ -50,89 +57,184 @@ class RouteOpt: return True return False + def getLinksName(self, routes,initial_start_edge,initial_end_edge, mappingTable): + routes=list(routes) + arr=routes[0]['x'] + listOfLinks=[] + for i in range(0, len(routes[0]['x'])): + if arr[i] == 1 : + # listOfLinks.append(self.fetchLogicalLinks(initial_start_edge[i], initial_end_edge[i], mappingTable)) + listOfLinks.append(mappingTable[initial_start_edge[i] + ":" + initial_end_edge[i]]) + + return listOfLinks + + # def search(self, ip1, ip2, dic): + # if ip1 == "" or ip2 == "": + # return "" + # else: + # string = ip1 + ":" + ip2 + # return dic[string] + # + # def fetchLogicalLinks(self, initial_start_edge, initial_end_edge, mappingTable): + # link_name=self.search(initial_start_edge, initial_end_edge, mappingTable) + # return link_name + + + # def fetchLogicalLinks(self, initial_start_edge, initial_end_edge, mappingTable): + # return mappingTable[initial_start_edge + ":" + initial_end_edge] + + def solve(self, mzn_model, dzn_data): + return pymzn.minizinc(mzn=mzn_model, data=dzn_data) + + def getLinks(self, mzn_model, dzn_data, initial_start_edge,initial_end_edge, mappingTable): + routes = self.solve(mzn_model, dzn_data) + audit_log.info("mocked minizinc solution====>") + audit_log.info(routes) + + converted_links=self.getLinksName(routes, initial_start_edge,initial_end_edge, mappingTable) + audit_log.info("converted links===>") + audit_log.info(converted_links) + return converted_links + + def addition(self, data): + relationship = data["relationship-list"]["relationship"] + res = "" + for index, eachItem in enumerate(relationship): + if index == len(relationship) - 1: + res += eachItem["accessNodeId"] + else: + res += eachItem["accessNodeId"] + ":" + + return data["link-name"], res + + def createMapTable(self, logical_links): + result = map(self.addition, logical_links) + + parseTemplate = {} + + for eachItem in result: + parseTemplate[eachItem[1]] = eachItem[0] + audit_log.info("mapping table") + audit_log.info(parseTemplate) + return parseTemplate + + def build_dzn_data(self, src_access_node_id, dst_access_node_id): + Edge_Start = [] + Edge_End = [] + logical_links = self.get_logical_links() + audit_log.info("mocked response of AAI received (logical links) successful===>") + audit_log.info(logical_links) + # prepare map table + mappingTable = self.createMapTable(logical_links) + # take the logical link where both the p-interface in same onap + if logical_links is not None: + for logical_link in logical_links: + if not self.isCrossONAPLink(logical_link): + # link is in local ONAP + relationship = logical_link["relationship-list"]["relationship"] + + relationshipStartNode = relationship[0] + relationshipStartNodeID = relationshipStartNode["related-link"].split("/")[-1] + start_accessNodeId = relationshipStartNodeID.split("-")[-3] + Edge_Start.append(start_accessNodeId) + + relationshipEndtNode = relationship[1] + relationshipEndNodeID = relationshipEndtNode["related-link"].split("/")[-1] + end_accessNodeId = relationshipEndNodeID.split("-")[-3] + Edge_End.append(end_accessNodeId) + + audit_log.info("edge start and end array of i/p address are===>") + audit_log.info(Edge_Start) + audit_log.info(Edge_End) + # labeling ip to number for mapping + le = preprocessing.LabelEncoder() + le.fit(Edge_Start + Edge_End) + # print(le.classes_) + dzn_start_edge = le.transform(Edge_Start) + + final_dzn_start_arr = [] + for i in range(0, len(dzn_start_edge)): + final_dzn_start_arr.append(dzn_start_edge[i]) + + final_dzn_end_arr = [] + dzn_end_edge = le.transform(Edge_End) + for j in range(0, len(dzn_end_edge)): + final_dzn_end_arr.append(dzn_end_edge[j]) + + audit_log.info("start and end array that passed in dzn_data===>") + audit_log.info(final_dzn_start_arr) + audit_log.info(final_dzn_end_arr) + + link_cost = [] + for k in range(0, len(final_dzn_start_arr)): + link_cost.append(1) + + audit_log.info("src_access_node_id") + audit_log.info(src_access_node_id) + source= le.transform([src_access_node_id]) + audit_log.info("vallue of source===>") + audit_log.info(source) + if source in final_dzn_start_arr : + start = source[0] + audit_log.info("source node") + audit_log.info(start) + + audit_log.info("dst_access_node_id") + audit_log.info(dst_access_node_id) + destination= le.transform([dst_access_node_id]) + if destination in final_dzn_end_arr : + end = destination[0] + audit_log.info("destination node") + audit_log.info(end) + # data to be prepared in the below format: + dzn_data = { + 'N': self.total_node(final_dzn_start_arr + final_dzn_end_arr), + 'M': len(final_dzn_start_arr), + 'Edge_Start': final_dzn_start_arr, + 'Edge_End': final_dzn_end_arr, + 'L': link_cost, + 'Start': start, + 'End': end, + } + # can not do reverse mapping outside of this scope, so doing here + audit_log.info("reverse mapping after prepared dzn_data") + initial_start_edge=le.inverse_transform(final_dzn_start_arr) + initial_end_edge=le.inverse_transform(final_dzn_end_arr) + audit_log.info(initial_start_edge) + audit_log.info(initial_end_edge) + return dzn_data, initial_start_edge,initial_end_edge, mappingTable + + def total_node(self, node): + nodeSet = set() + for i in range(0, len(node)): + nodeSet.add(node[i]) + total_node = len(nodeSet) + return total_node + def getRoute(self, request): """ - This method checks + This method checks :param logical_link: :return: """ - mdc_from_json(request) - - src_access_node_id = request["srcPort"]["src-access-node-id"] - dst_access_node_id = request["dstPort"]["dst-access-node-id"] - - - ingress_p_interface = None - egress_p_interface = None - - # for the case of request_json for same domain, return the same node with destination update - if src_access_node_id == dst_access_node_id: - data = '{'\ - '"vpns":['\ - '{'\ - '"access-topology-id": "' + request["srcPort"]["src-access-topology-id"] + '",'\ - '"access-client-id": "' + request["srcPort"]["src-access-client-id"] + '",'\ - '"access-provider-id": "' + request["srcPort"]["src-access-provider-id"]+ '",'\ - '"access-node-id": "' + request["srcPort"]["src-access-node-id"]+ '",'\ - '"src-access-ltp-id": "' + request["srcPort"]["src-access-ltp-id"]+ '",'\ - '"dst-access-ltp-id": "' + request["dstPort"]["dst-access-ltp-id"] +'"'\ - '}'\ - ']'\ - '}' - return data - else: - logical_links = self.get_logical_links() - - # take the logical link where both the p-interface in same onap - if logical_links != None: - for logical_link in logical_links.get("logical-link"): - if not self.isCrossONAPLink(logical_link): - # link is in local ONAP - for relationship in logical_link["relationship-list"]["relationship"]: - if relationship["related-to"] == "p-interface": - if src_access_node_id in relationship["related-link"]: - i_interface = relationship["related-link"].split("/")[-1] - ingress_p_interface = i_interface.split("-")[-1] - if dst_access_node_id in relationship["related-link"]: - e_interface = relationship["related-link"].split("/")[-1] - egress_p_interface = e_interface.split("-")[-1] - data = '{'\ - '"vpns":['\ - '{'\ - '"access-topology-id": "' + request["srcPort"]["src-access-topology-id"] + '",'\ - '"access-client-id": "' + request["srcPort"]["src-access-client-id"] + '",'\ - '"access-provider-id": "' + request["srcPort"]["src-access-provider-id"]+ '",'\ - '"access-node-id": "' + request["srcPort"]["src-access-node-id"]+ '",'\ - '"src-access-ltp-id": "' + request["srcPort"]["src-access-ltp-id"]+ '",'\ - '"dst-access-ltp-id": "' + ingress_p_interface +'"'\ - '},'\ - '{' \ - '"access-topology-id": "' + request["dstPort"]["dst-access-topology-id"] + '",' \ - '"access-topology-id": "' + request["dstPort"]["dst-access-topology-id"]+ '",' \ - '"access-provider-id": "' + request["dstPort"]["dst-access-provider-id"]+ '",' \ - '"access-node-id": "' + request["dstPort"]["dst-access-node-id"]+ '",' \ - '"src-access-ltp-id": "' + egress_p_interface + '",' \ - '"dst-access-ltp-id": "' + request["dstPort"]["dst-access-ltp-id"] + '"' \ - '}'\ - ']'\ - '}' - return data - - - def get_pinterface(self, url): - """ - This method returns details for p interface - :return: details of p interface - """ - aai_req_url = self.aai_host + url - response = requests.get(aai_req_url, - headers=self.aai_headers, - auth=HTTPBasicAuth("AAI", "AAI"), - verify=False) + routeInfo = request["routeInfo"]["routeRequests"] + routeRequest = routeInfo[0] + src_access_node_id = routeRequest["srcPort"]["accessNodeId"] + dst_access_node_id = routeRequest["dstPort"]["accessNodeId"] - if response.status_code == 200: - return response.json() + dzn_data, initial_start_edge, initial_end_edge, mappingTable = self.build_dzn_data(src_access_node_id, dst_access_node_id ) + #mzn_model = "/home/root1/Videos/projects/osdf/test/functest/simulators/osdf/optimizers/routeopt/route_opt.mzn" + mzn_model = os.path.join(BASE_DIR, 'route_opt.mzn') + routeSolutions = self.getLinks(mzn_model, dzn_data, initial_start_edge,initial_end_edge, mappingTable) + + return { + "requestId": request["requestInfo"]["requestId"], + "transactionId": request["requestInfo"]["transactionId"], + "statusMessage": " ", + "requestStatus": "accepted", + "solutions": routeSolutions + } def get_logical_links(self): """ @@ -142,12 +244,6 @@ class RouteOpt: """ logical_link_url = "/aai/v13/network/logical-links?operational-status=up" aai_req_url = self.aai_host + logical_link_url - - response = requests.get(aai_req_url, - headers=self.aai_headers, - auth=HTTPBasicAuth("AAI", "AAI"), - verify=False) - - logical_links = None + response = requests.get(aai_req_url,headers=self.aai_headers,auth=HTTPBasicAuth("AAI", "AAI"),verify=False) if response.status_code == 200: return response.json()
\ No newline at end of file diff --git a/config/opteng_config.yaml b/config/opteng_config.yaml new file mode 100755 index 0000000..d6be7ed --- /dev/null +++ b/config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: http://localhost:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5
\ No newline at end of file diff --git a/config/osdf_config.yaml b/config/osdf_config.yaml index 6cf8cec..eba89e0 100755 --- a/config/osdf_config.yaml +++ b/config/osdf_config.yaml @@ -51,4 +51,7 @@ configDbGetCellListUrl: 'SDNCConfigDBAPI/getCellList' configDbGetNbrListUrl: 'SDNCConfigDBAPI/getNbrList' pciHMSUsername: test -pciHMSPassword: passwd
\ No newline at end of file +pciHMSPassword: passwd + +#key +appkey: os35@rrtky400fdntc#001t5
\ No newline at end of file diff --git a/config/preload_secrets.yaml b/config/preload_secrets.yaml index 3050d87..b95f1c1 100755 --- a/config/preload_secrets.yaml +++ b/config/preload_secrets.yaml @@ -1,51 +1,55 @@ --- domain: osdf secrets: -- name: so - values: - UserName: '' - Password: '' -- name: conductor - values: - UserName: admin1 - Password: plan.15 -- name: policyPlatform - values: - UserName: healthcheck - Password: zb!XztG34 -- name: dmaap - values: - UserName: NA - Password: NA -- name: sdc - values: - UserName: NA - Password: NA -- name: osdfPlacement - values: - UserName: test - Password: testpwd -- name: osdfPlacementSO - values: - UserName: so_test - Password: so_testpwd -- name: osdfPlacementVFC - values: - UserName: vfc_test - Password: vfc_testpwd -- name: osdfCMScheduler - values: - UserName: test1 - Password: testpwd1 -- name: configDb - values: - UserName: osdf - Password: passwd -- name: pciHMS - values: - UserName: '' - Password: '' -- name: osdfPCIOpt - values: - UserName: pci_test - Password: pci_testpwd + - name: so + values: + UserName: '' + Password: '' + - name: conductor + values: + UserName: admin1 + Password: 22234d3472ef5da8ecba5a096110a024f1db5cf195c665f910d558c9e83db19d + - name: policyPlatform + values: + UserName: healthcheck + Password: 49a03554e86ecdb8e9e224127791c579b44993b264549a333172af77c2ae95fc + - name: dmaap + values: + UserName: NA + Password: NA + - name: sdc + values: + UserName: NA + Password: NA + - name: osdfPlacement + values: + UserName: test + Password: c66b1570ae257375e500f9fe0e62b2a325466137ac5f29581e2e05cce1170212 + - name: osdfPlacementSO + values: + UserName: so_test + Password: 3d62d49b3e4ada38fd4146d2d82f4ba2f09345a46f15970cd439924c991b8202 + - name: osdfPlacementVFC + values: + UserName: vfc_test + Password: 1fb1cd581f96060d29ecad06be97151656bf29bce66bad587cd2fbaf5ea1e66d + - name: osdfCMScheduler + values: + UserName: test1 + Password: c5279fb02d7bac5269b1a644ac8e36f41f6ba7a2eae03dc469cb80d71811322b + - name: configDb + values: + UserName: osdf + Password: 40697f254409c2b97763892ecdeb50c847d605f5beb6f988f1c142a7e0344d0c + - name: pciHMS + values: + UserName: '' + Password: '' + - name: osdfPCIOpt + values: + UserName: pci_test + Password: fbf4dcb7f7cda8fdfb742838b0c90ae5bea249801f3f725fdc98941a6e4c347c + - name: osdfOptEngine + values: + UserName: opt_test + Password: 02946408ce6353d45540cd01d912686f19f48c3d8a955d5effdc14c6a43477e5 diff --git a/docker/opteng/Dockerfile b/docker/opteng/Dockerfile new file mode 100644 index 0000000..9dca3e7 --- /dev/null +++ b/docker/opteng/Dockerfile @@ -0,0 +1,74 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +FROM python:3.8-alpine + +ARG MVN_ARTIFACT_VERSION +ARG REPO +ARG HTTP_PROXY=${HTTP_PROXY} +ARG HTTPS_PROXY=${HTTPS_PROXY} + +ENV http_proxy $HTTP_PROXY +ENV https_proxy $HTTPS_PROXY + +ENV OSDF_PORT "8699" +EXPOSE ${OSDF_PORT} + +ENV MZN 2.4.2 +ENV MZN_BASENAME MiniZincIDE-${MZN}-bundle-linux +ENV MZN_GH_BASE https://github.com/MiniZinc/MiniZincIDE +ENV MZN_DL_URL ${MZN_GH_BASE}/releases/download/${MZN}/${MZN_BASENAME}-x86_64.tgz + +RUN apk update && apk upgrade \ + && apk --no-cache --update add --virtual build-deps openssl wget \ + && apk --no-cache --update add less ca-certificates bash libxslt-dev unzip \ + freetype freetype-dev libstdc++ build-base libc6-compat \ + && ln -s /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2 + +# Minizinc +RUN wget -q $MZN_DL_URL -O mz.tgz \ + && tar xzf mz.tgz \ + && mv $MZN_BASENAME /mz-dist \ + && rm mz.tgz \ + && echo PATH=/mz-dist/bin:$PATH >> ~/.bashrc + +ENV SHELL /bin/bash +ENV PATH /mz-dist:$PATH + +RUN addgroup -S onap && adduser -S -G onap onap + +# OSDF +WORKDIR /opt/osdf +#RUN wget -O /opt/osdf.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=releases&g=org.onap.optf.osdf&a=optf-osdf&e=zip&v=1.3.4" && \ +# unzip -q -o -B /opt/osdf.zip -d /opt/ && \ +# rm -f /opt/osdf.zip + +COPY onap-osdf-tm/optf-osdf-${MVN_ARTIFACT_VERSION}.zip /tmp/optf-osdf.zip +COPY onap-osdf-tm/runtime /opt/osdf/runtime +COPY onap-osdf-tm/requirements-opteng.txt . +RUN unzip -q -o -B /tmp/optf-osdf.zip -d /opt/ && rm -f /tmp/optf-osdf.zip +RUN mkdir -p /var/log/onap/optf/osdf/ \ + && chown onap:onap /var/log/onap -R \ + && chown onap:onap /opt/osdf -R + +RUN pip install --no-cache-dir -r requirements.txt -r requirements-opteng.txt + +USER onap + +CMD [ "/opt/osdf/osdfapp.sh", "-x", "solverapp.py", "-c", "/opt/osdf/config/opteng_config.yaml" ] diff --git a/docker/opteng/assembly/osdf-files.xml b/docker/opteng/assembly/osdf-files.xml new file mode 100644 index 0000000..60dd6cc --- /dev/null +++ b/docker/opteng/assembly/osdf-files.xml @@ -0,0 +1,55 @@ +<!-- + Copyright (C) 2020 AT&T Intellectual Property. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +--> +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd"> + <id>osdf-files</id> + + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + + <fileSets> + <fileSet> + <includes> + <include>${project.build.finalName}.zip</include> + </includes> + <directory>${project.build.directory}</directory> + <outputDirectory>/</outputDirectory> + </fileSet> + <fileSet> + <includes> + <include>runtime/**</include> + </includes> + <excludes> + <exclude>**/*.pyc</exclude> + <exclude>**/__pycache__/**</exclude> + </excludes> + <outputDirectory>/</outputDirectory> + </fileSet> + <fileSet> + <includes> + <include>requirements-opteng.txt</include> + </includes> + <outputDirectory>/</outputDirectory> + </fileSet> + + </fileSets> +</assembly> diff --git a/docker/Dockerfile b/docker/osdf/Dockerfile index 0f271c8..5860df2 100644 --- a/docker/Dockerfile +++ b/docker/osdf/Dockerfile @@ -30,7 +30,7 @@ ENV https_proxy $HTTPS_PROXY ENV OSDF_PORT "8699" EXPOSE ${OSDF_PORT} -ENV MZN 2.3.2 +ENV MZN 2.4.2 ENV MZN_BASENAME MiniZincIDE-${MZN}-bundle-linux ENV MZN_GH_BASE https://github.com/MiniZinc/MiniZincIDE ENV MZN_DL_URL ${MZN_GH_BASE}/releases/download/${MZN}/${MZN_BASENAME}-x86_64.tgz @@ -49,8 +49,7 @@ RUN wget -q $MZN_DL_URL -O /tmp/mz.tgz \ && tar xzf /tmp/mz.tgz \ && mv $MZN_BASENAME /mz-dist \ && rm /tmp/mz.tgz \ - && echo PATH=/mz-dist/bin:$PATH >> ~/.bashrc \ - && echo 'export LD_LIBRARY_PATH=/mz-dist/lib:LD_LIBRARY_PATH' >> ~/.bashrc + && echo PATH=/mz-dist/bin:$PATH >> ~/.bashrc ENV SHELL /bin/bash ENV PATH /mz-dist:$PATH @@ -60,11 +59,18 @@ WORKDIR /opt/osdf #RUN wget -O /opt/osdf.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=releases&g=org.onap.optf.osdf&a=optf-osdf&e=zip&v=1.3.4" && \ # unzip -q -o -B /opt/osdf.zip -d /opt/ && \ # rm -f /opt/osdf.zip +RUN groupadd onap \ + && useradd -m -g onap onap COPY onap-osdf-tm/optf-osdf-${MVN_ARTIFACT_VERSION}.zip /tmp/optf-osdf.zip COPY onap-osdf-tm/apps /opt/osdf/apps RUN unzip -q -o -B /tmp/optf-osdf.zip -d /opt/ && rm -f /tmp/optf-osdf.zip -RUN mkdir -p /var/log/onap/optf/osdf/ +RUN mkdir -p /var/log/onap/optf/osdf/ \ + && chown -R onap:onap /var/log/onap \ + && chown -R onap:onap /opt/osdf + RUN pip install --no-cache-dir -r requirements.txt -CMD [ "/opt/osdf/osdfapp.sh" ] +USER onap + +CMD [ "/opt/osdf/osdfapp.sh", "-x", "osdfapp.py" ] diff --git a/docker/assembly/osdf-files.xml b/docker/osdf/assembly/osdf-files.xml index fc8a864..fc8a864 100644 --- a/docker/assembly/osdf-files.xml +++ b/docker/osdf/assembly/osdf-files.xml diff --git a/docker/build_image.sh b/docker/osdf/build_image.sh index 3d9d823..3d9d823 100755 --- a/docker/build_image.sh +++ b/docker/osdf/build_image.sh diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 0000000..43ca5b6 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +/.tox +/_build/* +/__pycache__/* diff --git a/docs/_static/css/ribbon.css b/docs/_static/css/ribbon.css new file mode 100644 index 0000000..6008cb1 --- /dev/null +++ b/docs/_static/css/ribbon.css @@ -0,0 +1,63 @@ +.ribbon { + z-index: 1000; + background-color: #a00; + overflow: hidden; + white-space: nowrap; + position: fixed; + top: 25px; + right: -50px; + -webkit-transform: rotate(45deg); + -moz-transform: rotate(45deg); + -ms-transform: rotate(45deg); + -o-transform: rotate(45deg); + transform: rotate(45deg); + -webkit-box-shadow: 0 0 10px #888; + -moz-box-shadow: 0 0 10px #888; + box-shadow: 0 0 10px #888; + +} + +.ribbon a { + border: 1px solid #faa; + color: #fff; + display: block; + font: bold 81.25% 'Helvetica Neue', Helvetica, Arial, sans-serif; + margin: 1px 0; + padding: 10px 50px; + text-align: center; + text-decoration: none; + text-shadow: 0 0 5px #444; + transition: 0.5s; +} + +.ribbon a:hover { + background: #c11; + color: #fff; +} + + +/* override table width restrictions */ +@media screen and (min-width: 767px) { + + .wy-table-responsive table td, .wy-table-responsive table th { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } + + .wy-table-responsive { + overflow: visible !important; + } +} + +@media screen and (max-width: 767px) { + .wy-table-responsive table td { + white-space: nowrap; + } +} + +/* fix width of the screen */ + +.wy-nav-content { + max-width: none; +} diff --git a/docs/_static/favicon.ico b/docs/_static/favicon.ico Binary files differnew file mode 100755 index 0000000..cb712eb --- /dev/null +++ b/docs/_static/favicon.ico diff --git a/docs/_static/logo_onap_2017.png b/docs/_static/logo_onap_2017.png Binary files differnew file mode 100644 index 0000000..5d064f4 --- /dev/null +++ b/docs/_static/logo_onap_2017.png diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..8f40e8b --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,15 @@ +from docs_conf.conf import * + +branch = 'latest' +master_doc = 'index' + +linkcheck_ignore = [ + 'http://localhost', +] + +intersphinx_mapping = {} + +html_last_updated_fmt = '%d-%b-%y %H:%M' + +def setup(app): + app.add_stylesheet("css/ribbon_onap.css") diff --git a/docs/conf.yaml b/docs/conf.yaml new file mode 100644 index 0000000..ab59281 --- /dev/null +++ b/docs/conf.yaml @@ -0,0 +1,7 @@ +--- +project_cfg: onap +project: onap + +# Change this to ReleaseBranchName to modify the header +default-version: latest +# diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt new file mode 100644 index 0000000..b3188dd --- /dev/null +++ b/docs/requirements-docs.txt @@ -0,0 +1,15 @@ +tox +Sphinx +doc8 +docutils +setuptools +six +sphinx_rtd_theme>=0.4.3 +sphinxcontrib-blockdiag +sphinxcontrib-needs>=0.2.3 +sphinxcontrib-nwdiag +sphinxcontrib-seqdiag +sphinxcontrib-swaggerdoc +sphinxcontrib-plantuml +sphinx_bootstrap_theme +lfdocs-conf diff --git a/docs/tox.ini b/docs/tox.ini new file mode 100644 index 0000000..edac8c3 --- /dev/null +++ b/docs/tox.ini @@ -0,0 +1,22 @@ +[tox] +minversion = 1.6 +envlist = docs, +skipsdist = true + +[testenv:docs] +basepython = python3 +deps = -r{toxinidir}/requirements-docs.txt +commands = + sphinx-build -b html -n -d {envtmpdir}/doctrees ./ {toxinidir}/_build/html + echo "Generated docs available in {toxinidir}/_build/html" +whitelist_externals = + echo + git + sh + +[testenv:docs-linkcheck] +basepython = python3 +#deps = -r{toxinidir}/requirements-docs.txt +commands = echo "Link Checking not enforced" +#commands = sphinx-build -b linkcheck -d {envtmpdir}/doctrees ./ {toxinidir}/_build/linkcheck +whitelist_externals = echo diff --git a/osdf/__init__.py b/osdf/__init__.py index c33639e..8036d89 100755 --- a/osdf/__init__.py +++ b/osdf/__init__.py @@ -20,11 +20,12 @@ from jinja2 import Template - end_point_auth_mapping = { # map a URL endpoint to auth group "cmscheduler": "CMScheduler", "placement": "Placement", - "pci": "PCIOpt" + "pci": "PCIOpt", + "optmodel": "OptEngine", + "optengine": "OptEngine" } userid_suffix, passwd_suffix = "Username", "Password" diff --git a/osdf/adapters/aaf/sms.py b/osdf/adapters/aaf/sms.py index 25ae7f2..0168ba0 100644 --- a/osdf/adapters/aaf/sms.py +++ b/osdf/adapters/aaf/sms.py @@ -1,6 +1,7 @@ # # ------------------------------------------------------------------------- # Copyright (c) 2018 Intel Corporation Intellectual Property +# Copyright (C) 2020 Wipro Limited. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,12 +21,12 @@ '''Secret Management Service Integration''' from onapsmsclient import Client - import osdf.config.base as cfg_base import osdf.config.credentials as creds import osdf.config.loader as config_loader from osdf.config.base import osdf_config from osdf.logging.osdf_logging import debug_log +from osdf.utils import cipherUtils config_spec = { "preload_secrets": "config/preload_secrets.yaml" @@ -70,40 +71,48 @@ def retrieve_secrets(): debug_log.debug("Secret Dictionary Retrieval Success") return secret_dict - def load_secrets(): config = osdf_config.deployment secret_dict = retrieve_secrets() config['soUsername'] = secret_dict['so']['UserName'] - config['soPassword'] = secret_dict['so']['Password'] + config['soPassword'] = decrypt_pass(secret_dict['so']['Password']) config['conductorUsername'] = secret_dict['conductor']['UserName'] - config['conductorPassword'] = secret_dict['conductor']['Password'] + config['conductorPassword'] = decrypt_pass(secret_dict['conductor']['Password']) config['policyPlatformUsername'] = secret_dict['policyPlatform']['UserName'] - config['policyPlatformPassword'] = secret_dict['policyPlatform']['Password'] - config['policyClientUsername'] = secret_dict['policyClient']['UserName'] - config['policyClientPassword'] = secret_dict['policyClient']['Password'] + config['policyPlatformPassword'] = decrypt_pass(secret_dict['policyPlatform']['Password']) + config['policyClientUsername'] = secret_dict['policyPlatform']['UserName'] + config['policyClientPassword'] = decrypt_pass(secret_dict['policyPlatform']['Password']) config['messageReaderAafUserId'] = secret_dict['dmaap']['UserName'] - config['messageReaderAafPassword'] = secret_dict['dmaap']['Password'] + config['messageReaderAafPassword'] = decrypt_pass(secret_dict['dmaap']['Password']) config['sdcUsername'] = secret_dict['sdc']['UserName'] - config['sdcPassword'] = secret_dict['sdc']['Password'] + config['sdcPassword'] = decrypt_pass(secret_dict['sdc']['Password']) config['osdfPlacementUsername'] = secret_dict['osdfPlacement']['UserName'] - config['osdfPlacementPassword'] = secret_dict['osdfPlacement']['Password'] + config['osdfPlacementPassword'] = decrypt_pass(secret_dict['osdfPlacement']['Password']) config['osdfPlacementSOUsername'] = secret_dict['osdfPlacementSO']['UserName'] - config['osdfPlacementSOPassword'] = secret_dict['osdfPlacementSO']['Password'] + config['osdfPlacementSOPassword'] = decrypt_pass(secret_dict['osdfPlacementSO']['Password']) config['osdfPlacementVFCUsername'] = secret_dict['osdfPlacementVFC']['UserName'] - config['osdfPlacementVFCPassword'] = secret_dict['osdfPlacementVFC']['Password'] + config['osdfPlacementVFCPassword'] = decrypt_pass(secret_dict['osdfPlacementVFC']['Password']) config['osdfCMSchedulerUsername'] = secret_dict['osdfCMScheduler']['UserName'] - config['osdfCMSchedulerPassword'] = secret_dict['osdfCMScheduler']['Password'] + config['osdfCMSchedulerPassword'] = decrypt_pass(secret_dict['osdfCMScheduler']['Password']) config['configDbUserName'] = secret_dict['configDb']['UserName'] - config['configDbPassword'] = secret_dict['configDb']['Password'] + config['configDbPassword'] = decrypt_pass(secret_dict['configDb']['Password']) config['pciHMSUsername'] = secret_dict['pciHMS']['UserName'] - config['pciHMSPassword'] = secret_dict['pciHMS']['Password'] + config['pciHMSPassword'] = decrypt_pass(secret_dict['pciHMS']['Password']) config['osdfPCIOptUsername'] = secret_dict['osdfPCIOpt']['UserName'] - config['osdfPCIOptPassword'] = secret_dict['osdfPCIOpt']['Password'] + config['osdfPCIOptPassword'] = decrypt_pass(secret_dict['osdfPCIOpt']['Password']) + config['osdfOptEngineUsername'] = secret_dict['osdfOptEngine']['UserName'] + config['osdfOptEnginePassword'] = decrypt_pass(secret_dict['osdfOptEngine']['Password']) cfg_base.http_basic_auth_credentials = creds.load_credentials(osdf_config) cfg_base.dmaap_creds = creds.dmaap_creds() +def decrypt_pass(passwd): + if passwd == '' or passwd == 'NA': + return passwd + else: + return cipherUtils.AESCipher.get_instance().decrypt(passwd) + + def delete_secrets(): """ This is intended to delete the secrets for a clean initialization for testing Application. Actual deployment will have a preload script. diff --git a/osdf/apps/baseapp.py b/osdf/apps/baseapp.py index cfa7e5d..fd94c11 100644 --- a/osdf/apps/baseapp.py +++ b/osdf/apps/baseapp.py @@ -27,18 +27,17 @@ import time import traceback from optparse import OptionParser -import pydevd -from flask import Flask, request, Response, g -from requests import RequestException -from schematics.exceptions import DataError - import osdf.adapters.aaf.sms as sms import osdf.operation.responses +import pydevd +from flask import Flask, request, Response, g from osdf.config.base import osdf_config from osdf.logging.osdf_logging import error_log, debug_log from osdf.operation.error_handling import request_exception_to_json_body, internal_error_message from osdf.operation.exceptions import BusinessException -from osdf.utils.mdc_utils import clear_mdc, mdc_from_json, default_mdc +from osdf.utils.mdc_utils import clear_mdc, mdc_from_json, default_mdc, get_request_id +from requests import RequestException +from schematics.exceptions import DataError ERROR_TEMPLATE = osdf.ERROR_TEMPLATE @@ -89,18 +88,20 @@ def handle_data_error(e): @app.before_request def log_request(): - g.request_start = time.clock() - if request.get_json(): - - request_json = request.get_json() - g.request_id = request_json['requestInfo']['requestId'] - mdc_from_json(request_json) + g.request_start = time.process_time() + if request.data: + if request.get_json(): + request_json = request.get_json() + g.request_id = get_request_id(request_json) + mdc_from_json(request_json) + else: + g.request_id = "N/A" + default_mdc() else: g.request_id = "N/A" default_mdc() - @app.after_request def log_response(response): clear_mdc() diff --git a/osdf/cmd/encryptionUtil.py b/osdf/cmd/encryptionUtil.py new file mode 100644 index 0000000..6c0cae2 --- /dev/null +++ b/osdf/cmd/encryptionUtil.py @@ -0,0 +1,50 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2015-2018 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# +import sys +from osdf.utils import cipherUtils + + +def main(): + + if len(sys.argv) != 4: + print("Invalid input - usage --> (options(encrypt/decrypt) input-value with-key)") + return + + enc_dec = sys.argv[1] + valid_option_values = ['encrypt', 'decrypt'] + if enc_dec not in valid_option_values: + print("Invalid input - usage --> (options(encrypt/decrypt) input-value with-key)") + print("Option value can only be one of {}".format(valid_option_values)) + print("You entered '{}'".format(enc_dec)) + return + + input_string = sys.argv[2] + with_key = sys.argv[3] + + print("You've requested '{}' to be '{}ed' using key '{}'".format(input_string, enc_dec, with_key)) + print("You can always perform the reverse operation (encrypt/decrypt) using the same key" + "to be certain you get the same results back'") + + util = cipherUtils.AESCipher.get_instance(with_key) + if enc_dec.lower() == 'encrypt': + result = util.encrypt(input_string) + else: + result = util.decrypt(input_string) + + print("Your resultt: {}".format(result))
\ No newline at end of file diff --git a/osdf/optimizers/pciopt/__init__.py b/osdf/optimizers/pciopt/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/osdf/optimizers/pciopt/__init__.py +++ /dev/null diff --git a/osdf/utils/cipherUtils.py b/osdf/utils/cipherUtils.py new file mode 100644 index 0000000..169f1a1 --- /dev/null +++ b/osdf/utils/cipherUtils.py @@ -0,0 +1,59 @@ +# +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- + +from Crypto.Cipher import AES +from osdf.config.base import osdf_config +from Crypto.Util.Padding import unpad +from Crypto.Util.Padding import pad + + +class AESCipher(object): + __instance = None + + @staticmethod + def get_instance(key = None): + if AESCipher.__instance is None: + print("Creating the singleton instance") + AESCipher(key) + return AESCipher.__instance + + def __init__(self, key=None): + if AESCipher.__instance is not None: + raise Exception("This class is a singleton!") + else: + AESCipher.__instance = self + + self.bs = 32 + if key is None: + key = osdf_config.deployment["appkey"] + + self.key = key.encode() + + def encrypt(self, data): + data = data.encode() + cipher = AES.new(self.key, AES.MODE_CBC) + ciphered_data = cipher.encrypt(pad(data, AES.block_size)) + enc = (cipher.iv.hex())+(ciphered_data.hex()) + return enc + + def decrypt(self, enc): + iv = bytes.fromhex(enc[:32]) + ciphered_data = bytes.fromhex(enc[32:]) + cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) + original_data = unpad(cipher.decrypt(ciphered_data), AES.block_size).decode() + return original_data diff --git a/osdf/utils/file_utils.py b/osdf/utils/file_utils.py new file mode 100644 index 0000000..b12c17d --- /dev/null +++ b/osdf/utils/file_utils.py @@ -0,0 +1,34 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +# File related utilities + +import os +from shutil import rmtree + +from osdf.logging.osdf_logging import debug_log + + +def delete_file_folder(p): + if not p: + return + debug_log.debug('Deleting folder/file {}'.format(p)) + if os.path.isfile(p): + os.remove(p) + else: + rmtree(p, ignore_errors=True) diff --git a/osdf/utils/mdc_utils.py b/osdf/utils/mdc_utils.py index b98cbf0..14b726d 100644 --- a/osdf/utils/mdc_utils.py +++ b/osdf/utils/mdc_utils.py @@ -36,7 +36,7 @@ def default_server_info(): MDC.put('server', server) if MDC.get('serverIPAddress') is None: try: - server_ip_address = socket.gethostbyname(self._fields['server']) + server_ip_address = socket.gethostbyname(MDC.get('server')) except Exception: server_ip_address = "" MDC.put('serverIPAddress', server_ip_address) @@ -53,9 +53,16 @@ def default_mdc(): def mdc_from_json(request_json): default_mdc() - MDC.put('requestID', request_json['requestInfo']['requestId']) + MDC.put('requestID', get_request_id(request_json)) MDC.put('partnerName', request_json['requestInfo']['sourceId']) +def get_request_id(request_json): + request_id = request_json['requestInfo'].get('requestId') + if not request_id: + request_id = request_json['requestInfo'].get('requestID') + return request_id + + def clear_mdc(): MDC.clear() diff --git a/osdf/webapp/appcontroller.py b/osdf/webapp/appcontroller.py index 9714fb5..5db879a 100644 --- a/osdf/webapp/appcontroller.py +++ b/osdf/webapp/appcontroller.py @@ -16,14 +16,16 @@ # ------------------------------------------------------------------------- # +import json + +from flask import Response from flask import request from flask_httpauth import HTTPBasicAuth -from flask import Response -import json + import osdf import osdf.config.base as cfg_base -from osdf.config.base import osdf_config from osdf.adapters.aaf import aaf_authentication as aaf_auth +from osdf.config.base import osdf_config auth_basic = HTTPBasicAuth() @@ -35,12 +37,15 @@ error_body = { unauthorized_message = json.dumps(error_body) + @auth_basic.get_password def get_pw(username): - end_point = request.url.split('/')[-1] - auth_group = osdf.end_point_auth_mapping.get(end_point) - return cfg_base.http_basic_auth_credentials[auth_group].get( - username) if auth_group else None + auth_group = '' + for k in osdf.end_point_auth_mapping: + if k in request.url: + auth_group = osdf.end_point_auth_mapping.get(k) + return cfg_base.http_basic_auth_credentials[auth_group].get(username) if auth_group else None + @auth_basic.error_handler def auth_error(): @@ -58,4 +63,3 @@ def verify_pw(username, password): else: pw = get_pw(username) return pw == password - return False
\ No newline at end of file @@ -21,6 +21,11 @@ OSDF Manager Main Flask Application """ import json +import ssl +import sys +import time +import traceback +from optparse import OptionParser from threading import Thread # for scaling up, may need celery with RabbitMQ or redis from flask import request, g @@ -89,14 +94,15 @@ def placement_rest_api(): version_info=api_version_info, request_status="accepted", status_message="") -@app.route("/api/oof/v1/route", methods=["POST"]) +@app.route("/api/oof/route/v1", methods=["POST"]) def do_route_calc(): """ Perform the basic route calculations and returnn the vpn-bindings """ request_json = request.get_json() audit_log.info("Calculate Route request received!") - return RouteOpt().getRoute(request_json) + response = RouteOpt().getRoute(request_json) + return response @app.route("/api/oof/v1/pci", methods=["POST"]) @@ -104,7 +110,13 @@ def do_route_calc(): @auth_basic.login_required def do_pci_optimization(): request_json = request.get_json() + audit_log.info('request json obtained==>') + audit_log.info(request_json) + req_id = request_json['requestInfo']['requestId'] + audit_log.info('requestID obtained==>') + audit_log.info(req_id) + g.request_id = req_id audit_log.info(MH.received_request(request.url, request.remote_addr, json.dumps(request_json))) PCIOptimizationAPI(request_json).validate() @@ -114,6 +126,7 @@ def do_pci_optimization(): t = Thread(target=process_pci_optimation, args=(request_json, osdf_config, None)) t.start() audit_log.info(MH.accepted_valid_request(req_id, request)) + audit_log.info('reached upto return') return req_accept(request_id=req_id, transaction_id=request_json['requestInfo']['transactionId'], request_status="accepted", status_message="") @@ -18,16 +18,46 @@ # ------------------------------------------------------------------------- # +usage() { + echo "Usage:" + echo " $0 -h Display this help message." + echo " $0 -c configfile_path(optional) -x app.py file" + exit 0 +} + cd $(dirname $0) # bash ../etc/make-certs.sh # create the https certificates if they are not present +while getopts ":hc:x:" opt; do + case ${opt} in + h ) + usage + ;; + c ) + # process option configuration + export OSDF_CONFIG_FILE=$OPTARG + ;; + x ) + # process executable file + export EXEC_FILE=$OPTARG + ;; + ? ) + usage + ;; + : ) + echo "Invalid Option: -$OPTARG requires an argument" 1>&2 + exit 1 + ;; + esac +done +shift $(( OPTIND - 1 )) + +set -e + LOGS=logs mkdir -p $LOGS -export OSDF_CONFIG_FILE=${1:-/opt/app/config/osdf_config.yaml} # this file may be passed by invoker -[ ! -e "$OSDF_CONFIG_FILE" ] && unset OSDF_CONFIG_FILE - if [ -e /opt/app/ssl_cert/aaf_root_ca.cer ]; then #assuming that this would be an ubuntu vm. cp /opt/app/ssl_cert/aaf_root_ca.cer /usr/local/share/ca-certificates/aafcacert.crt @@ -41,4 +71,11 @@ else export REQUESTS_CA_BUNDLE=/opt/app/ssl_cert/aaf_root_ca.cer fi -python osdfapp.py 2>$LOGS/err.log 1>$LOGS/out.log < /dev/null # running the app +if [ ! -z "$EXEC_FILE" ] +then + # flask run + echo "Running $EXEC_FILE" + python $EXEC_FILE # running the app +else + usage +fi @@ -14,77 +14,80 @@ License for the specific language governing permissions and limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <packaging>pom</packaging> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd +http://maven.apache.org/POM/4.0.0 "> + <modelVersion>4.0.0</modelVersion> + <packaging>pom</packaging> - <parent> - <groupId>org.onap.oparent</groupId> - <artifactId>oparent-python</artifactId> - <version>2.1.0</version> - </parent> + <parent> + <groupId>org.onap.oparent</groupId> + <artifactId>oparent-python</artifactId> + <version>3.0.0</version> + </parent> - <groupId>org.onap.optf.osdf</groupId> - <artifactId>optf-osdf</artifactId> - <name>optf-osdf</name> - <version>1.3.4-SNAPSHOT</version> - <description>Optimization Service Design Framework</description> + <groupId>org.onap.optf.osdf</groupId> + <artifactId>optf-osdf</artifactId> + <name>optf-osdf</name> + <version>1.3.4-SNAPSHOT</version> + <description>Optimization Service Design Framework</description> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <sonar.sources>.</sonar.sources> - <sonar.junit.reportsPath>xunit-results.xml</sonar.junit.reportsPath> - <sonar.python.coverage.reportPath>coverage.xml</sonar.python.coverage.reportPath> - <sonar.language>py</sonar.language> - <sonar.pluginname>python</sonar.pluginname> - <sonar.inclusions>**/**.py,osdfapp.py</sonar.inclusions> - <sonar.exclusions>test/**.py</sonar.exclusions> - <maven.build.timestamp.format>yyyyMMdd'T'HHmmss'Z'</maven.build.timestamp.format> - <osdf.build.timestamp>${maven.build.timestamp}</osdf.build.timestamp> - <osdf.project.version>${project.version}</osdf.project.version> - <osdf.docker.repository>nexus3.onap.org:10003</osdf.docker.repository> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <sonar.sources>.</sonar.sources> + <sonar.junit.reportsPath>xunit-results.xml</sonar.junit.reportsPath> + <sonar.python.coverage.reportPaths>coverage.xml</sonar.python.coverage.reportPaths> + <sonar.language>py</sonar.language> + <sonar.pluginname>python</sonar.pluginname> + <sonar.inclusions>**/**.py,osdfapp.py</sonar.inclusions> + <sonar.exclusions>test/**.py,docs/**.py</sonar.exclusions> + <maven.build.timestamp.format>yyyyMMdd'T'HHmmss'Z'</maven.build.timestamp.format> + <osdf.build.timestamp>${maven.build.timestamp}</osdf.build.timestamp> + <osdf.project.version>${project.version}</osdf.project.version> + <osdf.docker.repository>nexus3.onap.org:10003</osdf.docker.repository> <image.namespace>${osdf.docker.repository}/onap/optf-osdf</image.namespace> - </properties> + <opteng.namespace>${osdf.docker.repository}/onap/optf-opteng</opteng.namespace> + </properties> - <build> - <plugins> - <!-- triggers tox test for sonar --> - <plugin> - <artifactId>exec-maven-plugin</artifactId> - <groupId>org.codehaus.mojo</groupId> - </plugin> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <appendAssemblyId>false</appendAssemblyId> - <descriptors> - <descriptor>assembly.xml</descriptor> - </descriptors> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> + <build> + <plugins> + <!-- triggers tox test for sonar --> + <plugin> + <artifactId>exec-maven-plugin</artifactId> + <groupId>org.codehaus.mojo</groupId> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-release-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <version>2.8</version> - <configuration> - <retryFailedDeploymentCount>2</retryFailedDeploymentCount> - </configuration> - </plugin> - <plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-release-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8</version> + <configuration> + <retryFailedDeploymentCount>2</retryFailedDeploymentCount> + </configuration> + </plugin> + <plugin> <groupId>org.codehaus.groovy.maven</groupId> <artifactId>gmaven-plugin</artifactId> <version>1.0</version> @@ -100,7 +103,7 @@ </execution> </executions> </plugin> - <plugin> + <plugin> <groupId>io.fabric8</groupId> <artifactId>docker-maven-plugin</artifactId> <version>0.26.0</version> @@ -119,14 +122,14 @@ <tag>${project.docker.latesttag.version}</tag> </tags> - <dockerFile>${project.basedir}/docker/Dockerfile</dockerFile> + <dockerFile>${project.basedir}/docker/osdf/Dockerfile</dockerFile> <assembly> - <descriptor>${project.basedir}/docker/assembly/osdf-files.xml</descriptor> + <descriptor>${project.basedir}/docker/osdf/assembly/osdf-files.xml</descriptor> <name>onap-osdf-tm</name> </assembly> <args> - <MVN_ARTIFACT_VERSION>${project.version}</MVN_ARTIFACT_VERSION> - <REPO>${project.repo}</REPO> + <MVN_ARTIFACT_VERSION>${project.version}</MVN_ARTIFACT_VERSION> + <REPO>${project.repo}</REPO> <!-- plugin cannot handle empty (no proxy) arguments <http_proxy_arg>${docker.http_proxy}</http_proxy_arg> @@ -135,6 +138,33 @@ </args> </build> </image> + <image> + <name>${opteng.namespace}</name> + <alias>optf-opteng</alias> + <build> + <cleanup>true</cleanup> + <tags> + <tag>latest</tag> + <tag>${project.docker.latesttagtimestamp.version}</tag> + <tag>${project.docker.latesttag.version}</tag> + </tags> + + <dockerFile>${project.basedir}/docker/opteng/Dockerfile</dockerFile> + <assembly> + <descriptor>${project.basedir}/docker/opteng/assembly/osdf-files.xml</descriptor> + <name>onap-osdf-tm</name> + </assembly> + <args> + <MVN_ARTIFACT_VERSION>${project.version}</MVN_ARTIFACT_VERSION> + <REPO>${project.repo}</REPO> + + <!-- plugin cannot handle empty (no proxy) arguments + <http_proxy_arg>${docker.http_proxy}</http_proxy_arg> + <https_proxy_arg>${docker.https_proxy}</https_proxy_arg> + --> + </args> + </build> + </image> </images> </configuration> <executions> @@ -158,6 +188,6 @@ </execution> </executions> </plugin> - </plugins> - </build> + </plugins> + </build> </project> diff --git a/requirements-opteng.txt b/requirements-opteng.txt new file mode 100644 index 0000000..6d0b524 --- /dev/null +++ b/requirements-opteng.txt @@ -0,0 +1 @@ +mysql-connector-python>=8.0.12
\ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8001016..c3749e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ pydevd==1.4.0 onapsmsclient>=0.0.4 pymzn>=0.18.3 onappylog>=1.0.9 +pathtools>=0.1.2 +pycryptodome>=3.9.6 diff --git a/osdf/optimizers/routeopt/__init__.py b/runtime/__init__.py index c235f2a..2aa67d8 100644 --- a/osdf/optimizers/routeopt/__init__.py +++ b/runtime/__init__.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------- -# Copyright (c) 2018 Huawei Intellectual Property +# Copyright (c) 2020 AT&T Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/runtime/model_api.py b/runtime/model_api.py new file mode 100644 index 0000000..fd87333 --- /dev/null +++ b/runtime/model_api.py @@ -0,0 +1,215 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import traceback + +import mysql.connector +from flask import g, Flask, Response + +from osdf.config.base import osdf_config +from osdf.logging.osdf_logging import debug_log, error_log +from osdf.operation.exceptions import BusinessException + + +def init_db(): + if is_db_enabled(): + get_db() + + +def get_db(): + """Opens a new database connection if there is none yet for the + current application context. + """ + if not hasattr(g, 'pg'): + properties = osdf_config['deployment'] + host, db_port, db = properties["osdfDatabaseHost"], properties["osdfDatabasePort"], \ + properties.get("osdfDatabaseSchema") + user, password = properties["osdfDatabaseUsername"], properties["osdfDatabasePassword"] + g.pg = mysql.connector.connect(host=host, port=db_port, user=user, password=password, database=db) + return g.pg + + +def close_db(): + """Closes the database again at the end of the request.""" + if hasattr(g, 'pg'): + g.pg.close() + + +app = Flask(__name__) + + +def create_model_data(model_api): + with app.app_context(): + try: + model_info = model_api['modelInfo'] + model_id = model_info['modelId'] + debug_log.debug( + "persisting model_api {}".format(model_id)) + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id FROM optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + if cursor.fetchone() is None: + query = "INSERT INTO optim_model_data (model_id, model_content, description, solver_type) VALUES " \ + "(%s, %s, %s, %s)" + values = (model_id, model_info['modelContent'], model_info.get('description'), model_info['solver']) + cursor.execute(query, values) + g.pg.commit() + + debug_log.debug("A record successfully inserted for request_id: {}".format(model_id)) + return retrieve_model_data(model_id) + close_db() + else: + query = "UPDATE optim_model_data SET model_content = %s, description = %s, solver_type = %s where " \ + "model_id = %s " + values = (model_info['modelContent'], model_info.get('description'), model_info['solver'], model_id) + cursor.execute(query, values) + g.pg.commit() + + return retrieve_model_data(model_id) + close_db() + except Exception as err: + error_log.error("error for request_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + raise BusinessException(err) + + +def retrieve_model_data(model_id): + status, resp_data = get_model_data(model_id) + + if status == 200: + resp = json.dumps(build_model_dict(resp_data)) + return build_response(resp, status) + else: + resp = json.dumps({ + 'modelId': model_id, + 'statusMessage': "Error retrieving the model data for model {} due to {}".format(model_id, resp_data) + }) + return build_response(resp, status) + + +def build_model_dict(resp_data, content_needed=True): + resp = {'modelId': resp_data[0], 'description': resp_data[2] if resp_data[2] else '', + 'solver': resp_data[3]} + if content_needed: + resp.update({'modelContent': resp_data[1]}) + return resp + + +def build_response(resp, status): + response = Response(resp, content_type='application/json; charset=utf-8') + response.headers.add('content-length', len(resp)) + response.status_code = status + return response + + +def delete_model_data(model_id): + with app.app_context(): + try: + debug_log.debug("deleting model data given model_id = {}".format(model_id)) + d = dict(); + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "delete from optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + g.pg.commit() + close_db() + resp = { + "statusMessage": "model data for modelId {} deleted".format(model_id) + } + return build_response(json.dumps(resp), 200) + except Exception as err: + error_log.error("error deleting model_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + raise BusinessException(err) + + +def get_model_data(model_id): + with app.app_context(): + try: + debug_log.debug("getting model data given model_id = {}".format(model_id)) + d = dict(); + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id, model_content, description, solver_type FROM optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + if cursor is None: + return 400, "FAILED" + else: + rows = cursor.fetchone() + if rows is not None: + index = 0 + for row in rows: + d[index] = row + index = index + 1 + return 200, d + else: + close_db() + return 500, "NOT_FOUND" + except Exception: + error_log.error("error for request_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + return 500, "FAILED" + + +def retrieve_all_models(): + status, resp_data = get_all_models() + model_list = [] + if status == 200: + for r in resp_data: + model_list.append(build_model_dict(r, False)) + resp = json.dumps(model_list) + return build_response(resp, status) + + else: + resp = json.dumps({ + 'statusMessage': "Error retrieving all the model data due to {}".format(resp_data) + }) + return build_response(resp, status) + + +def get_all_models(): + with app.app_context(): + try: + debug_log.debug("getting all model data".format()) + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id, model_content, description, solver_type FROM optim_model_data" + + cursor.execute(query) + if cursor is None: + return 400, "FAILED" + else: + rows = cursor.fetchall() + if rows is not None: + return 200, rows + else: + close_db() + return 500, "NOT_FOUND" + except Exception: + error_log.error("error for request_id: {}".format(traceback.format_exc())) + close_db() + return 500, "FAILED" + + +def is_db_enabled(): + return osdf_config['deployment'].get('isDatabaseEnabled', False) diff --git a/osdf/optimizers/placementopt/__init__.py b/runtime/models/__init__.py index 4b25e5b..2aa67d8 100644 --- a/osdf/optimizers/placementopt/__init__.py +++ b/runtime/models/__init__.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------- -# Copyright (c) 2017-2018 AT&T Intellectual Property +# Copyright (c) 2020 AT&T Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/osdf/optimizers/__init__.py b/runtime/models/api/__init__.py index 4b25e5b..2aa67d8 100644 --- a/osdf/optimizers/__init__.py +++ b/runtime/models/api/__init__.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------- -# Copyright (c) 2017-2018 AT&T Intellectual Property +# Copyright (c) 2020 AT&T Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/runtime/models/api/model_request.py b/runtime/models/api/model_request.py new file mode 100644 index 0000000..710da4b --- /dev/null +++ b/runtime/models/api/model_request.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType +from schematics.types.compound import ModelType + +from osdf.models.api.common import OSDFModel + + +class RequestInfo(OSDFModel): + """Info for northbound request from client such as PCI-mS Handler""" + transactionId = StringType(required=True) + requestID = StringType(required=True) + sourceId = StringType(required=True) + + +class OptimModelInfo(OSDFModel): + """Optimizer request info details.""" + # ModelId from the database + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType(required=True) + # Description of the model + description = StringType() + # a large blob string containing the model (which is not that + # problematic since models are fairly small). + modelContent = StringType() + + +class OptimModelRequestAPI(OSDFModel): + """Request for Optimizer API (specific to optimization and additional metadata""" + requestInfo = ModelType(RequestInfo, required=True) + modelInfo = ModelType(OptimModelInfo, required=True) diff --git a/runtime/models/api/model_response.py b/runtime/models/api/model_response.py new file mode 100644 index 0000000..e4a41a5 --- /dev/null +++ b/runtime/models/api/model_response.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType + +from osdf.models.api.common import OSDFModel + + +class OptimModelResponse(OSDFModel): + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType() + # a large blob string containing the model + modelContent = StringType() + # statusMessage + statusMessage = StringType() diff --git a/runtime/models/api/optim_request.py b/runtime/models/api/optim_request.py new file mode 100644 index 0000000..4a046d2 --- /dev/null +++ b/runtime/models/api/optim_request.py @@ -0,0 +1,60 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import BaseType, DictType, StringType, IntType +from schematics.types.compound import ModelType + +from osdf.models.api.common import OSDFModel + +""" +""" +class RequestInfo(OSDFModel): + """Info for northbound request from client """ + transactionId = StringType(required=True) + requestID = StringType(required=True) + callbackUrl = StringType() + sourceId = StringType(required=True) + timeout = IntType() + + +class DataInfo(OSDFModel): + """Optimization data info""" + text = StringType() + json = DictType(BaseType) + + +class OptimInfo(OSDFModel): + """Optimizer request info details.""" + # ModelId from the database, if its not populated, + # assume that solverModel will be populated. + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType() + # Arguments for solver + solverArgs = DictType(BaseType) + # NOTE: a large blob string containing the model (which is not that + # problematic since models are fairly small). + modelContent = StringType() + # Data Payload, input data for the solver + optData = ModelType(DataInfo) + + +class OptimizationAPI(OSDFModel): + """Request for Optimizer API (specific to optimization and additional metadata""" + requestInfo = ModelType(RequestInfo, required=True) + optimInfo = ModelType(OptimInfo, required=True) diff --git a/runtime/models/api/optim_response.py b/runtime/models/api/optim_response.py new file mode 100644 index 0000000..6fd0f6b --- /dev/null +++ b/runtime/models/api/optim_response.py @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType, BaseType +from schematics.types.compound import DictType + +from osdf.models.api.common import OSDFModel + + +class OptimResponse(OSDFModel): + transactionId = StringType(required=True) + requestID = StringType(required=True) + requestStatus = StringType(required=True) + statusMessage = StringType() + solutions = DictType(BaseType) diff --git a/runtime/optim_engine.py b/runtime/optim_engine.py new file mode 100644 index 0000000..4a8788e --- /dev/null +++ b/runtime/optim_engine.py @@ -0,0 +1,79 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from flask import Response + +from osdf.operation.exceptions import BusinessException +from .model_api import get_model_data +from .models.api.optim_request import OptimizationAPI +from .solvers.mzn.mzn_solver import solve as mzn_solve +from .solvers.py.py_solver import solve as py_solve + + +def is_valid_optim_request(request_json): + # Method to check whether the requestinfo/optimizer value is valid. + opt_info = request_json['optimInfo'] + if not opt_info.get('modelId'): + if not opt_info.get('modelContent') or not opt_info.get('solver'): + raise BusinessException('modelContent and solver needs to be populated if model_id is not set') + if not opt_info.get('optData'): + raise BusinessException('optimInfo.optData needs to be populated to solve for a problem') + + return True + + +def validate_request(request_json): + OptimizationAPI(request_json).validate() + if not is_valid_optim_request(request_json): + raise BusinessException('Invalid optim request ') + return True + + +def process_request(request_json): + response_code, response_message = run_optimizer(request_json) + response = Response(response_message, content_type='application/json; charset=utf-8') + response.headers.add('content-length', len(response_message)) + response.status_code = response_code + return response + + +def run_optimizer(request_json): + validate_request(request_json) + + model_content, solver = get_model_content(request_json) + + if solver == 'mzn': + return mzn_solve(request_json, model_content) + elif solver == 'py': + return py_solve(request_json, model_content) + raise BusinessException('Unsupported optimization solver requested {} '.format(solver)) + + +def get_model_content(request_json): + model_id = request_json['optimInfo'].get('modelId') + if model_id: + status, data = get_model_data(model_id) + if status == 200: + model_content = data[1] + solver = data[3] + else: + raise BusinessException('model_id [{}] not found in the model database'.format(model_id)) + else: + model_content = request_json['optimInfo']['modelContent'] + solver = request_json['optimInfo']['solver'] + return model_content, solver diff --git a/osdf/optimizers/licenseopt/__init__.py b/runtime/solvers/__init__.py index 4b25e5b..2aa67d8 100644 --- a/osdf/optimizers/licenseopt/__init__.py +++ b/runtime/solvers/__init__.py @@ -1,5 +1,5 @@ # ------------------------------------------------------------------------- -# Copyright (c) 2017-2018 AT&T Intellectual Property +# Copyright (c) 2020 AT&T Intellectual Property # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/runtime/solvers/mzn/__init__.py b/runtime/solvers/mzn/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/solvers/mzn/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/solvers/mzn/mzn_solver.py b/runtime/solvers/mzn/mzn_solver.py new file mode 100644 index 0000000..cf002e7 --- /dev/null +++ b/runtime/solvers/mzn/mzn_solver.py @@ -0,0 +1,102 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +from datetime import datetime + +from pymzn import Status, minizinc, cbc, gecode, chuffed, or_tools + +from osdf.utils.file_utils import delete_file_folder + +error_status_map = { + Status.INCOMPLETE: "incomplete", + Status.COMPLETE: "complete", + Status.UNSATISFIABLE: "unsatisfiable", + Status.UNKNOWN: "unknown", + Status.UNBOUNDED: "unbounded", + Status.UNSATorUNBOUNDED: "unsat_or_unbounded", + Status.ERROR: "error" +} + +solver_dict = { + 'cbc': cbc, + 'geocode': gecode, + 'chuffed': chuffed, + 'cp': chuffed, + 'or_tools': or_tools +} + + +def map_status(status): + return error_status_map.get(status, "failed") + + +def solve(request_json, mzn_content): + req_info = request_json['requestInfo'] + opt_info = request_json['optimInfo'] + try: + mzn_solution = mzn_solver(mzn_content, opt_info) + + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'done', + 'statusMessage': map_status(mzn_solution.status), + 'solutions': mzn_solution[0] if mzn_solution else {} + } + return 200, json.dumps(response) + except Exception as e: + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'failed', + 'statusMessage': 'Failed due to {}'.format(e) + } + return 400, json.dumps(response) + + +def mzn_solver(mzn_content, opt_info): + args = opt_info['solverArgs'] + solver = get_mzn_solver(args.pop('solver')) + mzn_opts = dict() + + try: + file_name = persist_opt_data(opt_info) + mzn_opts.update(args) + return minizinc(mzn_content, file_name, **mzn_opts, solver=solver) + + finally: + delete_file_folder(file_name) + + +def persist_opt_data(opt_info): + + if opt_info['optData'].get('json'): + data_content = json.dumps(opt_info['optData']['json']) + file_name = '/tmp/optim_engine_{}.json'.format(datetime.timestamp(datetime.now())) + elif opt_info['optData'].get('text'): + data_content = opt_info['optData']['text'] + file_name = '/tmp/optim_engine_{}.dzn'.format(datetime.timestamp(datetime.now())) + + with open(file_name, "wt") as data: + data.write(data_content) + return file_name + + +def get_mzn_solver(solver): + return solver_dict.get(solver) diff --git a/runtime/solvers/py/__init__.py b/runtime/solvers/py/__init__.py new file mode 100644 index 0000000..a8aa582 --- /dev/null +++ b/runtime/solvers/py/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +#
\ No newline at end of file diff --git a/runtime/solvers/py/py_solver.py b/runtime/solvers/py/py_solver.py new file mode 100644 index 0000000..6b200ab --- /dev/null +++ b/runtime/solvers/py/py_solver.py @@ -0,0 +1,92 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import subprocess +import traceback +from datetime import datetime + +from osdf.logging.osdf_logging import error_log, debug_log +from osdf.utils.file_utils import delete_file_folder + + +def py_solver(py_content, opt_info): + py_file = '/tmp/custom_heuristics_{}.py'.format(datetime.timestamp(datetime.now())) + with open(py_file, "wt") as f: + f.write(py_content) + if opt_info['optData'].get('json'): + data_content = json.dumps(opt_info['optData']['json']) + input_file = '/tmp/optim_engine_{}.json'.format(datetime.timestamp(datetime.now())) + elif opt_info['optData'].get('text'): + data_content = opt_info['optData']['text'] + input_file = '/tmp/optim_engine_{}.txt'.format(datetime.timestamp(datetime.now())) + with open(input_file, "wt") as f: + f.write(data_content) + + output_file = '/tmp/opteng_output_{}.json'.format(datetime.timestamp(datetime.now())) + + command = ['python', py_file, input_file, output_file] + + try: + p = subprocess.run(command, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + + debug_log.debug('Process return code {}'.format(p.returncode)) + if p.returncode > 0: + error_log.error('Process return code {} {}'.format(p.returncode, p.stdout)) + return 'error', {} + with open(output_file) as file: + data = file.read() + return 'success', json.loads(data) + + except Exception as e: + error_log.error("Error running optimizer {}".format(traceback.format_exc())) + return 'error', {} + finally: + cleanup((input_file, output_file, py_file)) + + +def cleanup(file_tup): + for f in file_tup: + try: + delete_file_folder(f) + except Exception as e: + error_log.error("Failed deleting the file {} - {}".format(f, traceback.format_exc())) + + +def solve(request_json, py_content): + req_info = request_json['requestInfo'] + opt_info = request_json['optimInfo'] + try: + status, solution = py_solver(py_content, opt_info) + + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': status, + 'statusMessage': "completed", + 'solutions': solution if solution else {} + } + return 200, json.dumps(response) + except Exception as e: + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'failed', + 'statusMessage': 'Failed due to {}'.format(e) + } + return 400, json.dumps(response) diff --git a/script/TagVersion.groovy b/script/TagVersion.groovy index 6ed6558..01bc840 100644 --- a/script/TagVersion.groovy +++ b/script/TagVersion.groovy @@ -37,7 +37,7 @@ if ( project.properties['osdf.project.version'].endsWith("-SNAPSHOT") ) { project.properties['project.docker.latesttagtimestamp.version']=versionTag + "-SNAPSHOT-"+timestamp; project.properties['project.repo'] = 'snapshots' } else { - project.properties['project.docker.latesttag.version']=baseTag + "-STAGING-latest"; + project.properties['project.docker.latesttag.version']=versionTag + "-STAGING-latest"; project.properties['project.docker.latesttagtimestamp.version']=versionTag + "-STAGING-"+timestamp; project.properties['project.repo'] = 'releases' } diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1dffa77 --- /dev/null +++ b/setup.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +# ------------------------------------------------------------------------- +# Copyright (C) 2020 Wipro Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +'''Setup''' + +import setuptools + +setuptools.setup(name='of-osdf', + version='1.0', + description='Python Distribution Utilities', + author='xyz', + author_email='xyz@wipro.com', + url='https://wiki.onap.org/display/DW/Optimization+Service+Design+Framework', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: ONAP', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3' + 'Programming Language :: Python :: 3.5' + 'Topic :: Communications :: Email', + 'Topic :: Office/Business', + 'Topic :: Software Development :: Bug Tracking',], + keywords=['onap','osdf'], + packages=['osdf'], + entry_points = { + 'console_scripts': [ + 'cipher-utility = osdf.cmd.encryptionUtil:main', + ], + 'oslo.config.opts': [ + 'osdf = osdf.opts:list_opts', + ], + } + ) diff --git a/solverapp.py b/solverapp.py new file mode 100644 index 0000000..39f2670 --- /dev/null +++ b/solverapp.py @@ -0,0 +1,81 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from flask import request, g + +from osdf.apps.baseapp import app, run_app +from osdf.logging.osdf_logging import audit_log +from osdf.webapp.appcontroller import auth_basic +from runtime.model_api import create_model_data, retrieve_model_data, retrieve_all_models, delete_model_data +from runtime.models.api.model_request import OptimModelRequestAPI +from runtime.optim_engine import process_request + + +@app.route("/api/oof/optengine/v1", methods=["POST"]) +@auth_basic.login_required +def opt_engine_rest_api(): + """Perform OptimEngine optimization after validating the request + """ + request_json = request.get_json() + return process_request(request_json) + + +@app.route("/api/oof/optmodel/v1", methods=["PUT", "POST"]) +@auth_basic.login_required +def opt_model_create_rest_api(): + """Perform OptimEngine optimization after validating the request + """ + request_json = request.get_json() + OptimModelRequestAPI(request_json).validate() + return create_model_data(request_json) + + +@app.route("/api/oof/optmodel/v1/<model_id>", methods=["GET"]) +@auth_basic.login_required +def opt_get_model_rest_api(model_id): + """Retrieve model data + """ + + return retrieve_model_data(model_id) + + +@app.route("/api/oof/optmodel/v1", methods=["GET"]) +@auth_basic.login_required +def opt_get_all_models_rest_api(): + """Retrieve all models data + """ + return retrieve_all_models() + + +@app.route("/api/oof/optmodel/v1/<model_id>", methods=["DELETE"]) +@auth_basic.login_required +def opt_delete_model_rest_api(model_id): + """Perform OptimEngine optimization after validating the request + """ + return delete_model_data(model_id) + + +@app.route("/api/oof/optengine/healthcheck/v1", methods=["GET"]) +def do_health_check(): + """Simple health check""" + audit_log.info("A OptimEngine health check v1 request is processed!") + return "OK" + + +if __name__ == "__main__": + run_app() diff --git a/test/config/opteng_config.yaml b/test/config/opteng_config.yaml new file mode 100755 index 0000000..4a7e57d --- /dev/null +++ b/test/config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: https://aaf-sms.onap:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5
\ No newline at end of file diff --git a/test/functest/simulators/simulated-config/opteng_config.yaml b/test/functest/simulators/simulated-config/opteng_config.yaml new file mode 100755 index 0000000..4a7e57d --- /dev/null +++ b/test/functest/simulators/simulated-config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: https://aaf-sms.onap:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5
\ No newline at end of file diff --git a/test/optengine-tests/test_modelapi_invalid.json b/test/optengine-tests/test_modelapi_invalid.json new file mode 100644 index 0000000..a58258e --- /dev/null +++ b/test/optengine-tests/test_modelapi_invalid.json @@ -0,0 +1,13 @@ +{ + "requestInfo": { + "transactinId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt" + }, + "modelInfo": { + "modelId": "model2", + "solver": "mzn", + "description": "graph coloring problem for australia", + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_modelapi_valid.json b/test/optengine-tests/test_modelapi_valid.json new file mode 100644 index 0000000..1fbca5b --- /dev/null +++ b/test/optengine-tests/test_modelapi_valid.json @@ -0,0 +1,13 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt" + }, + "modelInfo": { + "modelId": "model2", + "solver": "mzn", + "description": "graph coloring problem for australia", + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid.json b/test/optengine-tests/test_optengine_invalid.json new file mode 100644 index 0000000..9a0267a --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid.json @@ -0,0 +1,18 @@ +{ + "requestInfo": { + "transactioId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];", + "optData": { + "nc": 3 + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid2.json b/test/optengine-tests/test_optengine_invalid2.json new file mode 100644 index 0000000..23c5a8e --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid2.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + + "solverArgs": { + "solver": "cbc" + }, + "modelContent": "% Baking cakes for the school fete (with data file)\r\n\r\nint: flour; %no. grams of flour available\r\nint: banana; %no. of bananas available\r\nint: sugar; %no. grams of sugar available\r\nint: butter; %no. grams of butter available\r\nint: cocoa; %no. grams of cocoa available\r\n\r\nconstraint assert(flour >= 0,\"Invalid datafile: \" ++\r\n \"Amount of flour should be non-negative\");\r\nconstraint assert(banana >= 0,\"Invalid datafile: \" ++\r\n \"Amount of banana should be non-negative\");\r\nconstraint assert(sugar >= 0,\"Invalid datafile: \" ++\r\n \"Amount of sugar should be non-negative\");\r\nconstraint assert(butter >= 0,\"Invalid datafile: \" ++\r\n \"Amount of butter should be non-negative\");\r\nconstraint assert(cocoa >= 0,\"Invalid datafile: \" ++\r\n \"Amount of cocoa should be non-negative\");\r\n\r\nvar 0..100: b; % no. of banana cakes\r\nvar 0..100: c; % no. of chocolate cakes\r\n\r\n% flour\r\nconstraint 250*b + 200*c <= flour;\r\n% bananas\r\nconstraint 2*b <= banana;\r\n% sugar\r\nconstraint 75*b + 150*c <= sugar;\r\n% butter\r\nconstraint 100*b + 150*c <= butter;\r\n% cocoa\r\nconstraint 75*c <= cocoa;\r\n\r\n% maximize our profit\r\nsolve maximize 400*b + 450*c;\r\n\r\noutput [\"no. of banana cakes = \\(b)\\n\",\r\n \"no. of chocolate cakes = \\(c)\\n\"];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid_solver.json b/test/optengine-tests/test_optengine_invalid_solver.json new file mode 100644 index 0000000..a967c16 --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid_solver.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "apy", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_modelId.json b/test/optengine-tests/test_optengine_modelId.json new file mode 100644 index 0000000..b676d91 --- /dev/null +++ b/test/optengine-tests/test_optengine_modelId.json @@ -0,0 +1,19 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "modelId": "test", + "solverArgs": { + "solver": "geocode" + }, + "optData": { + "json": { + "nc": 3 + } + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_no_modelid.json b/test/optengine-tests/test_optengine_no_modelid.json new file mode 100644 index 0000000..9a8c3a4 --- /dev/null +++ b/test/optengine-tests/test_optengine_no_modelid.json @@ -0,0 +1,24 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "cbc" + }, + "modelContent": "% Baking cakes for the school fete (with data file)\r\n\r\nint: flour; %no. grams of flour available\r\nint: banana; %no. of bananas available\r\nint: sugar; %no. grams of sugar available\r\nint: butter; %no. grams of butter available\r\nint: cocoa; %no. grams of cocoa available\r\n\r\nconstraint assert(flour >= 0,\"Invalid datafile: \" ++\r\n \"Amount of flour should be non-negative\");\r\nconstraint assert(banana >= 0,\"Invalid datafile: \" ++\r\n \"Amount of banana should be non-negative\");\r\nconstraint assert(sugar >= 0,\"Invalid datafile: \" ++\r\n \"Amount of sugar should be non-negative\");\r\nconstraint assert(butter >= 0,\"Invalid datafile: \" ++\r\n \"Amount of butter should be non-negative\");\r\nconstraint assert(cocoa >= 0,\"Invalid datafile: \" ++\r\n \"Amount of cocoa should be non-negative\");\r\n\r\nvar 0..100: b; % no. of banana cakes\r\nvar 0..100: c; % no. of chocolate cakes\r\n\r\n% flour\r\nconstraint 250*b + 200*c <= flour;\r\n% bananas\r\nconstraint 2*b <= banana;\r\n% sugar\r\nconstraint 75*b + 150*c <= sugar;\r\n% butter\r\nconstraint 100*b + 150*c <= butter;\r\n% cocoa\r\nconstraint 75*c <= cocoa;\r\n\r\n% maximize our profit\r\nsolve maximize 400*b + 450*c;\r\n\r\noutput [\"no. of banana cakes = \\(b)\\n\",\r\n \"no. of chocolate cakes = \\(c)\\n\"];", + "optData": { + "json": { + "flour": 4000, + "banana": 6, + "sugar": 2000, + "butter": 500, + "cocoa": 500 + } + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_no_optdata.json b/test/optengine-tests/test_optengine_no_optdata.json new file mode 100644 index 0000000..f6645c8 --- /dev/null +++ b/test/optengine-tests/test_optengine_no_optdata.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_solverid.json b/test/optengine-tests/test_optengine_solverid.json new file mode 100644 index 0000000..bfd446c --- /dev/null +++ b/test/optengine-tests/test_optengine_solverid.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "py", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_valid.json b/test/optengine-tests/test_optengine_valid.json new file mode 100644 index 0000000..8de2b71 --- /dev/null +++ b/test/optengine-tests/test_optengine_valid.json @@ -0,0 +1,20 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];", + "optData": { + "json": { + "nc": 3 + } + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_py_optengine_valid.json b/test/optengine-tests/test_py_optengine_valid.json new file mode 100644 index 0000000..bfd446c --- /dev/null +++ b/test/optengine-tests/test_py_optengine_valid.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "py", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +}
\ No newline at end of file diff --git a/test/test_model_api.py b/test/test_model_api.py new file mode 100644 index 0000000..2a1cecf --- /dev/null +++ b/test/test_model_api.py @@ -0,0 +1,71 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import os + +import pytest +from mock import patch +from schematics.exceptions import DataError + +from runtime.model_api import create_model_data, get_model_data, delete_model_data, retrieve_all_models +from runtime.models.api.model_request import OptimModelRequestAPI +from runtime.optim_engine import validate_request + +BASE_DIR = os.path.dirname(__file__) + +ret_val = {'modelId': 'test', 'description': 'desc', 'solver': 'mzn'} + + +class TestModelApi(): + + def test_valid_mapi_request(self): + req_json = json.loads(open("./test/optengine-tests/test_modelapi_valid.json").read()) + + assert OptimModelRequestAPI(req_json).validate() is None + + def test_invalid_mapi_request(self): + req_json = json.loads(open("./test/optengine-tests/test_modelapi_invalid.json").read()) + with pytest.raises(DataError): + validate_request(req_json) + + @patch('runtime.model_api.build_model_dict') + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_create_model(self, config, conn, model_data): + model_data.return_value = ret_val + req_json = json.loads(open("./test/optengine-tests/test_modelapi_valid.json").read()) + + create_model_data(req_json) + + @patch('runtime.model_api.build_model_dict') + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_retrieve_model(self, config, conn, model_data): + model_data.return_value = ret_val + get_model_data('test') + + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_delete_model(self, config, conn): + delete_model_data('test') + + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_retrieve_all_model(self, config, conn): + retrieve_all_models() diff --git a/test/test_optim_engine.py b/test/test_optim_engine.py new file mode 100644 index 0000000..e1756f8 --- /dev/null +++ b/test/test_optim_engine.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import os + +import pytest +from mock import patch +from schematics.exceptions import DataError + +from osdf.operation.exceptions import BusinessException +from runtime.optim_engine import validate_request, process_request + +BASE_DIR = os.path.dirname(__file__) + + +class TestOptimEngine(): + + def test_valid_optim_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_valid.json").read()) + + assert validate_request(req_json) == True + + def test_invalid_optim_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid.json").read()) + with pytest.raises(DataError): + validate_request(req_json) + + def test_invalid_optim_request_without_modelid(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid2.json").read()) + with pytest.raises(BusinessException): + validate_request(req_json) + + def test_invalid_optim_request_no_optdata(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_no_optdata.json").read()) + with pytest.raises(BusinessException): + validate_request(req_json) + + def test_process_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_valid.json").read()) + + res = process_request(req_json) + assert res.status_code == 400 + + def test_py_process_request(self): + req_json = json.loads(open("./test/optengine-tests/test_py_optengine_valid.json").read()) + + res = process_request(req_json) + assert res.status_code == 200 + + def test_invalid_solver(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid_solver.json").read()) + + with pytest.raises(BusinessException): + process_request(req_json) + + @patch('runtime.optim_engine.get_model_data') + def test_process_solverid_request(self, mocker): + req_json = json.loads(open("./test/optengine-tests/test_optengine_modelId.json").read()) + + data = 200, ('junk', '', '', 'py') + mocker.return_value = data + process_request(req_json) @@ -5,6 +5,7 @@ envlist = py3, pylint [testenv] distribute = False +basepython=python3 setenv = OSDF_CONFIG_FILE={toxinidir}/test/config/osdf_config.yaml commands = @@ -16,13 +17,14 @@ commands = # TODO: need to update the above "omit" when we package osdf as pip-installable deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test/test-requirements.txt + -r{toxinidir}/requirements-opteng.txt [run] -source=./osdf/,osdfapp.py +source=./apps/,./osdf/,osdfapp.py,./runtime/,solverapp.py [testenv:pylint] whitelist_externals=bash -commands = bash -c "pylint --reports=y osdf | tee pylint.out" +commands = bash -c "pylint --reports=y osdf apps runtime| tee pylint.out" [testenv:py3] basepython=python3.6 |
