diff options
| author |   vrvarma <vikas.varma@att.com> | 2020-03-03 22:22:28 -0500 |
|---|---|---|
| committer | vrvarma <vikas.varma@att.com> | 2020-03-04 20:55:57 -0500 |
| commit | de5fdaafad9fccba0b9a7f308d72f26816dd1a0f (patch) | |
| tree | e8e683e21002c20f3d743375d742bf0819d82409 | |
| parent | 7d4f37c45d50c56dfe438c04dbecea3ca9f7c9d2 (diff) | |
Adding the generic solver code
Add docker file for optim engine
Run pods as a non-root user
Fix docker tag script
Change-Id: If25fe66b839a70e83e35292031a2da012e81fe47
Signed-off-by: vrvarma <vikas.varma@att.com>
Issue-ID: OPTFRA-712
49 files changed, 1597 insertions, 95 deletions
diff --git a/.coveragerc b/.coveragerc index a4ec20c..1fa0d3b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,7 @@ [run] branch = True cover_pylib = False -include = osdf/**/*.py, apps/**/*.py +include = osdf/**/*.py, apps/**/*.py, runtime/*.py, runtime/**/*.py [report] # Regexes for lines to exclude from consideration diff --git a/config/opteng_config.yaml b/config/opteng_config.yaml new file mode 100755 index 0000000..d6be7ed --- /dev/null +++ b/config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: http://localhost:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5
\ No newline at end of file diff --git a/config/preload_secrets.yaml b/config/preload_secrets.yaml index 0bb2395..b95f1c1 100755 --- a/config/preload_secrets.yaml +++ b/config/preload_secrets.yaml @@ -49,3 +49,7 @@ secrets: values: UserName: pci_test Password: fbf4dcb7f7cda8fdfb742838b0c90ae5bea249801f3f725fdc98941a6e4c347c + - name: osdfOptEngine + values: + UserName: opt_test + Password: 02946408ce6353d45540cd01d912686f19f48c3d8a955d5effdc14c6a43477e5 diff --git a/docker/opteng/Dockerfile b/docker/opteng/Dockerfile new file mode 100644 index 0000000..9dca3e7 --- /dev/null +++ b/docker/opteng/Dockerfile @@ -0,0 +1,74 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +FROM python:3.8-alpine + +ARG MVN_ARTIFACT_VERSION +ARG REPO +ARG HTTP_PROXY=${HTTP_PROXY} +ARG HTTPS_PROXY=${HTTPS_PROXY} + +ENV http_proxy $HTTP_PROXY +ENV https_proxy $HTTPS_PROXY + +ENV OSDF_PORT "8699" +EXPOSE ${OSDF_PORT} + +ENV MZN 2.4.2 +ENV MZN_BASENAME MiniZincIDE-${MZN}-bundle-linux +ENV MZN_GH_BASE https://github.com/MiniZinc/MiniZincIDE +ENV MZN_DL_URL ${MZN_GH_BASE}/releases/download/${MZN}/${MZN_BASENAME}-x86_64.tgz + +RUN apk update && apk upgrade \ + && apk --no-cache --update add --virtual build-deps openssl wget \ + && apk --no-cache --update add less ca-certificates bash libxslt-dev unzip \ + freetype freetype-dev libstdc++ build-base libc6-compat \ + && ln -s /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2 + +# Minizinc +RUN wget -q $MZN_DL_URL -O mz.tgz \ + && tar xzf mz.tgz \ + && mv $MZN_BASENAME /mz-dist \ + && rm mz.tgz \ + && echo PATH=/mz-dist/bin:$PATH >> ~/.bashrc + +ENV SHELL /bin/bash +ENV PATH /mz-dist:$PATH + +RUN addgroup -S onap && adduser -S -G onap onap + +# OSDF +WORKDIR /opt/osdf +#RUN wget -O /opt/osdf.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=releases&g=org.onap.optf.osdf&a=optf-osdf&e=zip&v=1.3.4" && \ +# unzip -q -o -B /opt/osdf.zip -d /opt/ && \ +# rm -f /opt/osdf.zip + +COPY onap-osdf-tm/optf-osdf-${MVN_ARTIFACT_VERSION}.zip /tmp/optf-osdf.zip +COPY onap-osdf-tm/runtime /opt/osdf/runtime +COPY onap-osdf-tm/requirements-opteng.txt . +RUN unzip -q -o -B /tmp/optf-osdf.zip -d /opt/ && rm -f /tmp/optf-osdf.zip +RUN mkdir -p /var/log/onap/optf/osdf/ \ + && chown onap:onap /var/log/onap -R \ + && chown onap:onap /opt/osdf -R + +RUN pip install --no-cache-dir -r requirements.txt -r requirements-opteng.txt + +USER onap + +CMD [ "/opt/osdf/osdfapp.sh", "-x", "solverapp.py", "-c", "/opt/osdf/config/opteng_config.yaml" ] diff --git a/docker/opteng/assembly/osdf-files.xml b/docker/opteng/assembly/osdf-files.xml new file mode 100644 index 0000000..60dd6cc --- /dev/null +++ b/docker/opteng/assembly/osdf-files.xml @@ -0,0 +1,55 @@ +<!-- + Copyright (C) 2020 AT&T Intellectual Property. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +--> +<assembly + xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd"> + <id>osdf-files</id> + + <formats> + <format>tar.gz</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + + <fileSets> + <fileSet> + <includes> + <include>${project.build.finalName}.zip</include> + </includes> + <directory>${project.build.directory}</directory> + <outputDirectory>/</outputDirectory> + </fileSet> + <fileSet> + <includes> + <include>runtime/**</include> + </includes> + <excludes> + <exclude>**/*.pyc</exclude> + <exclude>**/__pycache__/**</exclude> + </excludes> + <outputDirectory>/</outputDirectory> + </fileSet> + <fileSet> + <includes> + <include>requirements-opteng.txt</include> + </includes> + <outputDirectory>/</outputDirectory> + </fileSet> + + </fileSets> +</assembly> diff --git a/docker/Dockerfile b/docker/osdf/Dockerfile index e339ea7..5860df2 100644 --- a/docker/Dockerfile +++ b/docker/osdf/Dockerfile @@ -59,11 +59,18 @@ WORKDIR /opt/osdf #RUN wget -O /opt/osdf.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=releases&g=org.onap.optf.osdf&a=optf-osdf&e=zip&v=1.3.4" && \ # unzip -q -o -B /opt/osdf.zip -d /opt/ && \ # rm -f /opt/osdf.zip +RUN groupadd onap \ + && useradd -m -g onap onap COPY onap-osdf-tm/optf-osdf-${MVN_ARTIFACT_VERSION}.zip /tmp/optf-osdf.zip COPY onap-osdf-tm/apps /opt/osdf/apps RUN unzip -q -o -B /tmp/optf-osdf.zip -d /opt/ && rm -f /tmp/optf-osdf.zip -RUN mkdir -p /var/log/onap/optf/osdf/ +RUN mkdir -p /var/log/onap/optf/osdf/ \ + && chown -R onap:onap /var/log/onap \ + && chown -R onap:onap /opt/osdf + RUN pip install --no-cache-dir -r requirements.txt -CMD [ "/opt/osdf/osdfapp.sh" ] +USER onap + +CMD [ "/opt/osdf/osdfapp.sh", "-x", "osdfapp.py" ] diff --git a/docker/assembly/osdf-files.xml b/docker/osdf/assembly/osdf-files.xml index fc8a864..fc8a864 100644 --- a/docker/assembly/osdf-files.xml +++ b/docker/osdf/assembly/osdf-files.xml diff --git a/docker/build_image.sh b/docker/osdf/build_image.sh index 3d9d823..3d9d823 100755 --- a/docker/build_image.sh +++ b/docker/osdf/build_image.sh diff --git a/osdf/__init__.py b/osdf/__init__.py index c33639e..8036d89 100755 --- a/osdf/__init__.py +++ b/osdf/__init__.py @@ -20,11 +20,12 @@ from jinja2 import Template - end_point_auth_mapping = { # map a URL endpoint to auth group "cmscheduler": "CMScheduler", "placement": "Placement", - "pci": "PCIOpt" + "pci": "PCIOpt", + "optmodel": "OptEngine", + "optengine": "OptEngine" } userid_suffix, passwd_suffix = "Username", "Password" diff --git a/osdf/adapters/aaf/sms.py b/osdf/adapters/aaf/sms.py index fd3a5d5..0168ba0 100644 --- a/osdf/adapters/aaf/sms.py +++ b/osdf/adapters/aaf/sms.py @@ -100,6 +100,8 @@ def load_secrets(): config['pciHMSPassword'] = decrypt_pass(secret_dict['pciHMS']['Password']) config['osdfPCIOptUsername'] = secret_dict['osdfPCIOpt']['UserName'] config['osdfPCIOptPassword'] = decrypt_pass(secret_dict['osdfPCIOpt']['Password']) + config['osdfOptEngineUsername'] = secret_dict['osdfOptEngine']['UserName'] + config['osdfOptEnginePassword'] = decrypt_pass(secret_dict['osdfOptEngine']['Password']) cfg_base.http_basic_auth_credentials = creds.load_credentials(osdf_config) cfg_base.dmaap_creds = creds.dmaap_creds() diff --git a/osdf/apps/baseapp.py b/osdf/apps/baseapp.py index 008ce1d..fd94c11 100644 --- a/osdf/apps/baseapp.py +++ b/osdf/apps/baseapp.py @@ -35,7 +35,7 @@ from osdf.config.base import osdf_config from osdf.logging.osdf_logging import error_log, debug_log from osdf.operation.error_handling import request_exception_to_json_body, internal_error_message from osdf.operation.exceptions import BusinessException -from osdf.utils.mdc_utils import clear_mdc, mdc_from_json, default_mdc +from osdf.utils.mdc_utils import clear_mdc, mdc_from_json, default_mdc, get_request_id from requests import RequestException from schematics.exceptions import DataError @@ -88,11 +88,11 @@ def handle_data_error(e): @app.before_request def log_request(): - g.request_start = time.clock() + g.request_start = time.process_time() if request.data: if request.get_json(): request_json = request.get_json() - g.request_id = request_json['requestInfo']['requestId'] + g.request_id = get_request_id(request_json) mdc_from_json(request_json) else: g.request_id = "N/A" diff --git a/osdf/utils/file_utils.py b/osdf/utils/file_utils.py new file mode 100644 index 0000000..b12c17d --- /dev/null +++ b/osdf/utils/file_utils.py @@ -0,0 +1,34 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +# File related utilities + +import os +from shutil import rmtree + +from osdf.logging.osdf_logging import debug_log + + +def delete_file_folder(p): + if not p: + return + debug_log.debug('Deleting folder/file {}'.format(p)) + if os.path.isfile(p): + os.remove(p) + else: + rmtree(p, ignore_errors=True) diff --git a/osdf/utils/mdc_utils.py b/osdf/utils/mdc_utils.py index bcd0615..14b726d 100644 --- a/osdf/utils/mdc_utils.py +++ b/osdf/utils/mdc_utils.py @@ -53,9 +53,16 @@ def default_mdc(): def mdc_from_json(request_json): default_mdc() - MDC.put('requestID', request_json['requestInfo']['requestId']) + MDC.put('requestID', get_request_id(request_json)) MDC.put('partnerName', request_json['requestInfo']['sourceId']) +def get_request_id(request_json): + request_id = request_json['requestInfo'].get('requestId') + if not request_id: + request_id = request_json['requestInfo'].get('requestID') + return request_id + + def clear_mdc(): MDC.clear() diff --git a/osdf/webapp/appcontroller.py b/osdf/webapp/appcontroller.py index e48e93f..5db879a 100644 --- a/osdf/webapp/appcontroller.py +++ b/osdf/webapp/appcontroller.py @@ -16,14 +16,16 @@ # ------------------------------------------------------------------------- # +import json + +from flask import Response from flask import request from flask_httpauth import HTTPBasicAuth -from flask import Response -import json + import osdf import osdf.config.base as cfg_base -from osdf.config.base import osdf_config from osdf.adapters.aaf import aaf_authentication as aaf_auth +from osdf.config.base import osdf_config auth_basic = HTTPBasicAuth() @@ -38,10 +40,11 @@ unauthorized_message = json.dumps(error_body) @auth_basic.get_password def get_pw(username): - end_point = request.url.split('/')[-1] - auth_group = osdf.end_point_auth_mapping.get(end_point) - return cfg_base.http_basic_auth_credentials[auth_group].get( - username) if auth_group else None + auth_group = '' + for k in osdf.end_point_auth_mapping: + if k in request.url: + auth_group = osdf.end_point_auth_mapping.get(k) + return cfg_base.http_basic_auth_credentials[auth_group].get(username) if auth_group else None @auth_basic.error_handler @@ -18,16 +18,46 @@ # ------------------------------------------------------------------------- # +usage() { + echo "Usage:" + echo " $0 -h Display this help message." + echo " $0 -c configfile_path(optional) -x app.py file" + exit 0 +} + cd $(dirname $0) # bash ../etc/make-certs.sh # create the https certificates if they are not present +while getopts ":hc:x:" opt; do + case ${opt} in + h ) + usage + ;; + c ) + # process option configuration + export OSDF_CONFIG_FILE=$OPTARG + ;; + x ) + # process executable file + export EXEC_FILE=$OPTARG + ;; + ? ) + usage + ;; + : ) + echo "Invalid Option: -$OPTARG requires an argument" 1>&2 + exit 1 + ;; + esac +done +shift $(( OPTIND - 1 )) + +set -e + LOGS=logs mkdir -p $LOGS -export OSDF_CONFIG_FILE=${1:-/opt/app/config/osdf_config.yaml} # this file may be passed by invoker -[ ! -e "$OSDF_CONFIG_FILE" ] && unset OSDF_CONFIG_FILE - if [ -e /opt/app/ssl_cert/aaf_root_ca.cer ]; then #assuming that this would be an ubuntu vm. cp /opt/app/ssl_cert/aaf_root_ca.cer /usr/local/share/ca-certificates/aafcacert.crt @@ -41,4 +71,11 @@ else export REQUESTS_CA_BUNDLE=/opt/app/ssl_cert/aaf_root_ca.cer fi -python osdfapp.py 2>$LOGS/err.log 1>$LOGS/out.log < /dev/null # running the app +if [ ! -z "$EXEC_FILE" ] +then + # flask run + echo "Running $EXEC_FILE" + python $EXEC_FILE # running the app +else + usage +fi @@ -14,77 +14,80 @@ License for the specific language governing permissions and limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <packaging>pom</packaging> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd +http://maven.apache.org/POM/4.0.0 "> + <modelVersion>4.0.0</modelVersion> + <packaging>pom</packaging> - <parent> - <groupId>org.onap.oparent</groupId> - <artifactId>oparent-python</artifactId> - <version>3.0.0</version> - </parent> + <parent> + <groupId>org.onap.oparent</groupId> + <artifactId>oparent-python</artifactId> + <version>3.0.0</version> + </parent> - <groupId>org.onap.optf.osdf</groupId> - <artifactId>optf-osdf</artifactId> - <name>optf-osdf</name> - <version>1.3.4-SNAPSHOT</version> - <description>Optimization Service Design Framework</description> + <groupId>org.onap.optf.osdf</groupId> + <artifactId>optf-osdf</artifactId> + <name>optf-osdf</name> + <version>1.3.4-SNAPSHOT</version> + <description>Optimization Service Design Framework</description> - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <sonar.sources>.</sonar.sources> - <sonar.junit.reportsPath>xunit-results.xml</sonar.junit.reportsPath> - <sonar.python.coverage.reportPaths>coverage.xml</sonar.python.coverage.reportPaths> - <sonar.language>py</sonar.language> - <sonar.pluginname>python</sonar.pluginname> - <sonar.inclusions>**/**.py,osdfapp.py</sonar.inclusions> - <sonar.exclusions>test/**.py,docs/**.py</sonar.exclusions> - <maven.build.timestamp.format>yyyyMMdd'T'HHmmss'Z'</maven.build.timestamp.format> - <osdf.build.timestamp>${maven.build.timestamp}</osdf.build.timestamp> - <osdf.project.version>${project.version}</osdf.project.version> - <osdf.docker.repository>nexus3.onap.org:10003</osdf.docker.repository> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <sonar.sources>.</sonar.sources> + <sonar.junit.reportsPath>xunit-results.xml</sonar.junit.reportsPath> + <sonar.python.coverage.reportPaths>coverage.xml</sonar.python.coverage.reportPaths> + <sonar.language>py</sonar.language> + <sonar.pluginname>python</sonar.pluginname> + <sonar.inclusions>**/**.py,osdfapp.py</sonar.inclusions> + <sonar.exclusions>test/**.py,docs/**.py</sonar.exclusions> + <maven.build.timestamp.format>yyyyMMdd'T'HHmmss'Z'</maven.build.timestamp.format> + <osdf.build.timestamp>${maven.build.timestamp}</osdf.build.timestamp> + <osdf.project.version>${project.version}</osdf.project.version> + <osdf.docker.repository>nexus3.onap.org:10003</osdf.docker.repository> <image.namespace>${osdf.docker.repository}/onap/optf-osdf</image.namespace> - </properties> + <opteng.namespace>${osdf.docker.repository}/onap/optf-opteng</opteng.namespace> + </properties> - <build> - <plugins> - <!-- triggers tox test for sonar --> - <plugin> - <artifactId>exec-maven-plugin</artifactId> - <groupId>org.codehaus.mojo</groupId> - </plugin> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <configuration> - <appendAssemblyId>false</appendAssemblyId> - <descriptors> - <descriptor>assembly.xml</descriptor> - </descriptors> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> + <build> + <plugins> + <!-- triggers tox test for sonar --> + <plugin> + <artifactId>exec-maven-plugin</artifactId> + <groupId>org.codehaus.mojo</groupId> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>assembly.xml</descriptor> + </descriptors> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-release-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <version>2.8</version> - <configuration> - <retryFailedDeploymentCount>2</retryFailedDeploymentCount> - </configuration> - </plugin> - <plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-release-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-deploy-plugin</artifactId> + <version>2.8</version> + <configuration> + <retryFailedDeploymentCount>2</retryFailedDeploymentCount> + </configuration> + </plugin> + <plugin> <groupId>org.codehaus.groovy.maven</groupId> <artifactId>gmaven-plugin</artifactId> <version>1.0</version> @@ -100,7 +103,7 @@ </execution> </executions> </plugin> - <plugin> + <plugin> <groupId>io.fabric8</groupId> <artifactId>docker-maven-plugin</artifactId> <version>0.26.0</version> @@ -119,14 +122,14 @@ <tag>${project.docker.latesttag.version}</tag> </tags> - <dockerFile>${project.basedir}/docker/Dockerfile</dockerFile> + <dockerFile>${project.basedir}/docker/osdf/Dockerfile</dockerFile> <assembly> - <descriptor>${project.basedir}/docker/assembly/osdf-files.xml</descriptor> + <descriptor>${project.basedir}/docker/osdf/assembly/osdf-files.xml</descriptor> <name>onap-osdf-tm</name> </assembly> <args> - <MVN_ARTIFACT_VERSION>${project.version}</MVN_ARTIFACT_VERSION> - <REPO>${project.repo}</REPO> + <MVN_ARTIFACT_VERSION>${project.version}</MVN_ARTIFACT_VERSION> + <REPO>${project.repo}</REPO> <!-- plugin cannot handle empty (no proxy) arguments <http_proxy_arg>${docker.http_proxy}</http_proxy_arg> @@ -135,6 +138,33 @@ </args> </build> </image> + <image> + <name>${opteng.namespace}</name> + <alias>optf-opteng</alias> + <build> + <cleanup>true</cleanup> + <tags> + <tag>latest</tag> + <tag>${project.docker.latesttagtimestamp.version}</tag> + <tag>${project.docker.latesttag.version}</tag> + </tags> + + <dockerFile>${project.basedir}/docker/opteng/Dockerfile</dockerFile> + <assembly> + <descriptor>${project.basedir}/docker/opteng/assembly/osdf-files.xml</descriptor> + <name>onap-osdf-tm</name> + </assembly> + <args> + <MVN_ARTIFACT_VERSION>${project.version}</MVN_ARTIFACT_VERSION> + <REPO>${project.repo}</REPO> + + <!-- plugin cannot handle empty (no proxy) arguments + <http_proxy_arg>${docker.http_proxy}</http_proxy_arg> + <https_proxy_arg>${docker.https_proxy}</https_proxy_arg> + --> + </args> + </build> + </image> </images> </configuration> <executions> @@ -158,6 +188,6 @@ </execution> </executions> </plugin> - </plugins> - </build> + </plugins> + </build> </project> diff --git a/requirements-opteng.txt b/requirements-opteng.txt new file mode 100644 index 0000000..6d0b524 --- /dev/null +++ b/requirements-opteng.txt @@ -0,0 +1 @@ +mysql-connector-python>=8.0.12
\ No newline at end of file diff --git a/runtime/__init__.py b/runtime/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/model_api.py b/runtime/model_api.py new file mode 100644 index 0000000..fd87333 --- /dev/null +++ b/runtime/model_api.py @@ -0,0 +1,215 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import traceback + +import mysql.connector +from flask import g, Flask, Response + +from osdf.config.base import osdf_config +from osdf.logging.osdf_logging import debug_log, error_log +from osdf.operation.exceptions import BusinessException + + +def init_db(): + if is_db_enabled(): + get_db() + + +def get_db(): + """Opens a new database connection if there is none yet for the + current application context. + """ + if not hasattr(g, 'pg'): + properties = osdf_config['deployment'] + host, db_port, db = properties["osdfDatabaseHost"], properties["osdfDatabasePort"], \ + properties.get("osdfDatabaseSchema") + user, password = properties["osdfDatabaseUsername"], properties["osdfDatabasePassword"] + g.pg = mysql.connector.connect(host=host, port=db_port, user=user, password=password, database=db) + return g.pg + + +def close_db(): + """Closes the database again at the end of the request.""" + if hasattr(g, 'pg'): + g.pg.close() + + +app = Flask(__name__) + + +def create_model_data(model_api): + with app.app_context(): + try: + model_info = model_api['modelInfo'] + model_id = model_info['modelId'] + debug_log.debug( + "persisting model_api {}".format(model_id)) + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id FROM optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + if cursor.fetchone() is None: + query = "INSERT INTO optim_model_data (model_id, model_content, description, solver_type) VALUES " \ + "(%s, %s, %s, %s)" + values = (model_id, model_info['modelContent'], model_info.get('description'), model_info['solver']) + cursor.execute(query, values) + g.pg.commit() + + debug_log.debug("A record successfully inserted for request_id: {}".format(model_id)) + return retrieve_model_data(model_id) + close_db() + else: + query = "UPDATE optim_model_data SET model_content = %s, description = %s, solver_type = %s where " \ + "model_id = %s " + values = (model_info['modelContent'], model_info.get('description'), model_info['solver'], model_id) + cursor.execute(query, values) + g.pg.commit() + + return retrieve_model_data(model_id) + close_db() + except Exception as err: + error_log.error("error for request_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + raise BusinessException(err) + + +def retrieve_model_data(model_id): + status, resp_data = get_model_data(model_id) + + if status == 200: + resp = json.dumps(build_model_dict(resp_data)) + return build_response(resp, status) + else: + resp = json.dumps({ + 'modelId': model_id, + 'statusMessage': "Error retrieving the model data for model {} due to {}".format(model_id, resp_data) + }) + return build_response(resp, status) + + +def build_model_dict(resp_data, content_needed=True): + resp = {'modelId': resp_data[0], 'description': resp_data[2] if resp_data[2] else '', + 'solver': resp_data[3]} + if content_needed: + resp.update({'modelContent': resp_data[1]}) + return resp + + +def build_response(resp, status): + response = Response(resp, content_type='application/json; charset=utf-8') + response.headers.add('content-length', len(resp)) + response.status_code = status + return response + + +def delete_model_data(model_id): + with app.app_context(): + try: + debug_log.debug("deleting model data given model_id = {}".format(model_id)) + d = dict(); + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "delete from optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + g.pg.commit() + close_db() + resp = { + "statusMessage": "model data for modelId {} deleted".format(model_id) + } + return build_response(json.dumps(resp), 200) + except Exception as err: + error_log.error("error deleting model_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + raise BusinessException(err) + + +def get_model_data(model_id): + with app.app_context(): + try: + debug_log.debug("getting model data given model_id = {}".format(model_id)) + d = dict(); + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id, model_content, description, solver_type FROM optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + if cursor is None: + return 400, "FAILED" + else: + rows = cursor.fetchone() + if rows is not None: + index = 0 + for row in rows: + d[index] = row + index = index + 1 + return 200, d + else: + close_db() + return 500, "NOT_FOUND" + except Exception: + error_log.error("error for request_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + return 500, "FAILED" + + +def retrieve_all_models(): + status, resp_data = get_all_models() + model_list = [] + if status == 200: + for r in resp_data: + model_list.append(build_model_dict(r, False)) + resp = json.dumps(model_list) + return build_response(resp, status) + + else: + resp = json.dumps({ + 'statusMessage': "Error retrieving all the model data due to {}".format(resp_data) + }) + return build_response(resp, status) + + +def get_all_models(): + with app.app_context(): + try: + debug_log.debug("getting all model data".format()) + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id, model_content, description, solver_type FROM optim_model_data" + + cursor.execute(query) + if cursor is None: + return 400, "FAILED" + else: + rows = cursor.fetchall() + if rows is not None: + return 200, rows + else: + close_db() + return 500, "NOT_FOUND" + except Exception: + error_log.error("error for request_id: {}".format(traceback.format_exc())) + close_db() + return 500, "FAILED" + + +def is_db_enabled(): + return osdf_config['deployment'].get('isDatabaseEnabled', False) diff --git a/runtime/models/__init__.py b/runtime/models/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/models/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/models/api/__init__.py b/runtime/models/api/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/models/api/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/models/api/model_request.py b/runtime/models/api/model_request.py new file mode 100644 index 0000000..710da4b --- /dev/null +++ b/runtime/models/api/model_request.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType +from schematics.types.compound import ModelType + +from osdf.models.api.common import OSDFModel + + +class RequestInfo(OSDFModel): + """Info for northbound request from client such as PCI-mS Handler""" + transactionId = StringType(required=True) + requestID = StringType(required=True) + sourceId = StringType(required=True) + + +class OptimModelInfo(OSDFModel): + """Optimizer request info details.""" + # ModelId from the database + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType(required=True) + # Description of the model + description = StringType() + # a large blob string containing the model (which is not that + # problematic since models are fairly small). + modelContent = StringType() + + +class OptimModelRequestAPI(OSDFModel): + """Request for Optimizer API (specific to optimization and additional metadata""" + requestInfo = ModelType(RequestInfo, required=True) + modelInfo = ModelType(OptimModelInfo, required=True) diff --git a/runtime/models/api/model_response.py b/runtime/models/api/model_response.py new file mode 100644 index 0000000..e4a41a5 --- /dev/null +++ b/runtime/models/api/model_response.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType + +from osdf.models.api.common import OSDFModel + + +class OptimModelResponse(OSDFModel): + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType() + # a large blob string containing the model + modelContent = StringType() + # statusMessage + statusMessage = StringType() diff --git a/runtime/models/api/optim_request.py b/runtime/models/api/optim_request.py new file mode 100644 index 0000000..4a046d2 --- /dev/null +++ b/runtime/models/api/optim_request.py @@ -0,0 +1,60 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import BaseType, DictType, StringType, IntType +from schematics.types.compound import ModelType + +from osdf.models.api.common import OSDFModel + +""" +""" +class RequestInfo(OSDFModel): + """Info for northbound request from client """ + transactionId = StringType(required=True) + requestID = StringType(required=True) + callbackUrl = StringType() + sourceId = StringType(required=True) + timeout = IntType() + + +class DataInfo(OSDFModel): + """Optimization data info""" + text = StringType() + json = DictType(BaseType) + + +class OptimInfo(OSDFModel): + """Optimizer request info details.""" + # ModelId from the database, if its not populated, + # assume that solverModel will be populated. + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType() + # Arguments for solver + solverArgs = DictType(BaseType) + # NOTE: a large blob string containing the model (which is not that + # problematic since models are fairly small). + modelContent = StringType() + # Data Payload, input data for the solver + optData = ModelType(DataInfo) + + +class OptimizationAPI(OSDFModel): + """Request for Optimizer API (specific to optimization and additional metadata""" + requestInfo = ModelType(RequestInfo, required=True) + optimInfo = ModelType(OptimInfo, required=True) diff --git a/runtime/models/api/optim_response.py b/runtime/models/api/optim_response.py new file mode 100644 index 0000000..6fd0f6b --- /dev/null +++ b/runtime/models/api/optim_response.py @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType, BaseType +from schematics.types.compound import DictType + +from osdf.models.api.common import OSDFModel + + +class OptimResponse(OSDFModel): + transactionId = StringType(required=True) + requestID = StringType(required=True) + requestStatus = StringType(required=True) + statusMessage = StringType() + solutions = DictType(BaseType) diff --git a/runtime/optim_engine.py b/runtime/optim_engine.py new file mode 100644 index 0000000..4a8788e --- /dev/null +++ b/runtime/optim_engine.py @@ -0,0 +1,79 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from flask import Response + +from osdf.operation.exceptions import BusinessException +from .model_api import get_model_data +from .models.api.optim_request import OptimizationAPI +from .solvers.mzn.mzn_solver import solve as mzn_solve +from .solvers.py.py_solver import solve as py_solve + + +def is_valid_optim_request(request_json): + # Method to check whether the requestinfo/optimizer value is valid. + opt_info = request_json['optimInfo'] + if not opt_info.get('modelId'): + if not opt_info.get('modelContent') or not opt_info.get('solver'): + raise BusinessException('modelContent and solver needs to be populated if model_id is not set') + if not opt_info.get('optData'): + raise BusinessException('optimInfo.optData needs to be populated to solve for a problem') + + return True + + +def validate_request(request_json): + OptimizationAPI(request_json).validate() + if not is_valid_optim_request(request_json): + raise BusinessException('Invalid optim request ') + return True + + +def process_request(request_json): + response_code, response_message = run_optimizer(request_json) + response = Response(response_message, content_type='application/json; charset=utf-8') + response.headers.add('content-length', len(response_message)) + response.status_code = response_code + return response + + +def run_optimizer(request_json): + validate_request(request_json) + + model_content, solver = get_model_content(request_json) + + if solver == 'mzn': + return mzn_solve(request_json, model_content) + elif solver == 'py': + return py_solve(request_json, model_content) + raise BusinessException('Unsupported optimization solver requested {} '.format(solver)) + + +def get_model_content(request_json): + model_id = request_json['optimInfo'].get('modelId') + if model_id: + status, data = get_model_data(model_id) + if status == 200: + model_content = data[1] + solver = data[3] + else: + raise BusinessException('model_id [{}] not found in the model database'.format(model_id)) + else: + model_content = request_json['optimInfo']['modelContent'] + solver = request_json['optimInfo']['solver'] + return model_content, solver diff --git a/runtime/solvers/__init__.py b/runtime/solvers/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/solvers/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/solvers/mzn/__init__.py b/runtime/solvers/mzn/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/solvers/mzn/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/solvers/mzn/mzn_solver.py b/runtime/solvers/mzn/mzn_solver.py new file mode 100644 index 0000000..cf002e7 --- /dev/null +++ b/runtime/solvers/mzn/mzn_solver.py @@ -0,0 +1,102 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +from datetime import datetime + +from pymzn import Status, minizinc, cbc, gecode, chuffed, or_tools + +from osdf.utils.file_utils import delete_file_folder + +error_status_map = { + Status.INCOMPLETE: "incomplete", + Status.COMPLETE: "complete", + Status.UNSATISFIABLE: "unsatisfiable", + Status.UNKNOWN: "unknown", + Status.UNBOUNDED: "unbounded", + Status.UNSATorUNBOUNDED: "unsat_or_unbounded", + Status.ERROR: "error" +} + +solver_dict = { + 'cbc': cbc, + 'geocode': gecode, + 'chuffed': chuffed, + 'cp': chuffed, + 'or_tools': or_tools +} + + +def map_status(status): + return error_status_map.get(status, "failed") + + +def solve(request_json, mzn_content): + req_info = request_json['requestInfo'] + opt_info = request_json['optimInfo'] + try: + mzn_solution = mzn_solver(mzn_content, opt_info) + + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'done', + 'statusMessage': map_status(mzn_solution.status), + 'solutions': mzn_solution[0] if mzn_solution else {} + } + return 200, json.dumps(response) + except Exception as e: + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'failed', + 'statusMessage': 'Failed due to {}'.format(e) + } + return 400, json.dumps(response) + + +def mzn_solver(mzn_content, opt_info): + args = opt_info['solverArgs'] + solver = get_mzn_solver(args.pop('solver')) + mzn_opts = dict() + + try: + file_name = persist_opt_data(opt_info) + mzn_opts.update(args) + return minizinc(mzn_content, file_name, **mzn_opts, solver=solver) + + finally: + delete_file_folder(file_name) + + +def persist_opt_data(opt_info): + + if opt_info['optData'].get('json'): + data_content = json.dumps(opt_info['optData']['json']) + file_name = '/tmp/optim_engine_{}.json'.format(datetime.timestamp(datetime.now())) + elif opt_info['optData'].get('text'): + data_content = opt_info['optData']['text'] + file_name = '/tmp/optim_engine_{}.dzn'.format(datetime.timestamp(datetime.now())) + + with open(file_name, "wt") as data: + data.write(data_content) + return file_name + + +def get_mzn_solver(solver): + return solver_dict.get(solver) diff --git a/runtime/solvers/py/__init__.py b/runtime/solvers/py/__init__.py new file mode 100644 index 0000000..a8aa582 --- /dev/null +++ b/runtime/solvers/py/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +#
\ No newline at end of file diff --git a/runtime/solvers/py/py_solver.py b/runtime/solvers/py/py_solver.py new file mode 100644 index 0000000..6b200ab --- /dev/null +++ b/runtime/solvers/py/py_solver.py @@ -0,0 +1,92 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import subprocess +import traceback +from datetime import datetime + +from osdf.logging.osdf_logging import error_log, debug_log +from osdf.utils.file_utils import delete_file_folder + + +def py_solver(py_content, opt_info): + py_file = '/tmp/custom_heuristics_{}.py'.format(datetime.timestamp(datetime.now())) + with open(py_file, "wt") as f: + f.write(py_content) + if opt_info['optData'].get('json'): + data_content = json.dumps(opt_info['optData']['json']) + input_file = '/tmp/optim_engine_{}.json'.format(datetime.timestamp(datetime.now())) + elif opt_info['optData'].get('text'): + data_content = opt_info['optData']['text'] + input_file = '/tmp/optim_engine_{}.txt'.format(datetime.timestamp(datetime.now())) + with open(input_file, "wt") as f: + f.write(data_content) + + output_file = '/tmp/opteng_output_{}.json'.format(datetime.timestamp(datetime.now())) + + command = ['python', py_file, input_file, output_file] + + try: + p = subprocess.run(command, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + + debug_log.debug('Process return code {}'.format(p.returncode)) + if p.returncode > 0: + error_log.error('Process return code {} {}'.format(p.returncode, p.stdout)) + return 'error', {} + with open(output_file) as file: + data = file.read() + return 'success', json.loads(data) + + except Exception as e: + error_log.error("Error running optimizer {}".format(traceback.format_exc())) + return 'error', {} + finally: + cleanup((input_file, output_file, py_file)) + + +def cleanup(file_tup): + for f in file_tup: + try: + delete_file_folder(f) + except Exception as e: + error_log.error("Failed deleting the file {} - {}".format(f, traceback.format_exc())) + + +def solve(request_json, py_content): + req_info = request_json['requestInfo'] + opt_info = request_json['optimInfo'] + try: + status, solution = py_solver(py_content, opt_info) + + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': status, + 'statusMessage': "completed", + 'solutions': solution if solution else {} + } + return 200, json.dumps(response) + except Exception as e: + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'failed', + 'statusMessage': 'Failed due to {}'.format(e) + } + return 400, json.dumps(response) diff --git a/script/TagVersion.groovy b/script/TagVersion.groovy index 6ed6558..01bc840 100644 --- a/script/TagVersion.groovy +++ b/script/TagVersion.groovy @@ -37,7 +37,7 @@ if ( project.properties['osdf.project.version'].endsWith("-SNAPSHOT") ) { project.properties['project.docker.latesttagtimestamp.version']=versionTag + "-SNAPSHOT-"+timestamp; project.properties['project.repo'] = 'snapshots' } else { - project.properties['project.docker.latesttag.version']=baseTag + "-STAGING-latest"; + project.properties['project.docker.latesttag.version']=versionTag + "-STAGING-latest"; project.properties['project.docker.latesttagtimestamp.version']=versionTag + "-STAGING-"+timestamp; project.properties['project.repo'] = 'releases' } diff --git a/solverapp.py b/solverapp.py new file mode 100644 index 0000000..39f2670 --- /dev/null +++ b/solverapp.py @@ -0,0 +1,81 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from flask import request, g + +from osdf.apps.baseapp import app, run_app +from osdf.logging.osdf_logging import audit_log +from osdf.webapp.appcontroller import auth_basic +from runtime.model_api import create_model_data, retrieve_model_data, retrieve_all_models, delete_model_data +from runtime.models.api.model_request import OptimModelRequestAPI +from runtime.optim_engine import process_request + + +@app.route("/api/oof/optengine/v1", methods=["POST"]) +@auth_basic.login_required +def opt_engine_rest_api(): + """Perform OptimEngine optimization after validating the request + """ + request_json = request.get_json() + return process_request(request_json) + + +@app.route("/api/oof/optmodel/v1", methods=["PUT", "POST"]) +@auth_basic.login_required +def opt_model_create_rest_api(): + """Perform OptimEngine optimization after validating the request + """ + request_json = request.get_json() + OptimModelRequestAPI(request_json).validate() + return create_model_data(request_json) + + +@app.route("/api/oof/optmodel/v1/<model_id>", methods=["GET"]) +@auth_basic.login_required +def opt_get_model_rest_api(model_id): + """Retrieve model data + """ + + return retrieve_model_data(model_id) + + +@app.route("/api/oof/optmodel/v1", methods=["GET"]) +@auth_basic.login_required +def opt_get_all_models_rest_api(): + """Retrieve all models data + """ + return retrieve_all_models() + + +@app.route("/api/oof/optmodel/v1/<model_id>", methods=["DELETE"]) +@auth_basic.login_required +def opt_delete_model_rest_api(model_id): + """Perform OptimEngine optimization after validating the request + """ + return delete_model_data(model_id) + + +@app.route("/api/oof/optengine/healthcheck/v1", methods=["GET"]) +def do_health_check(): + """Simple health check""" + audit_log.info("A OptimEngine health check v1 request is processed!") + return "OK" + + +if __name__ == "__main__": + run_app() diff --git a/test/config/opteng_config.yaml b/test/config/opteng_config.yaml new file mode 100755 index 0000000..4a7e57d --- /dev/null +++ b/test/config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: https://aaf-sms.onap:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5
\ No newline at end of file diff --git a/test/functest/simulators/simulated-config/opteng_config.yaml b/test/functest/simulators/simulated-config/opteng_config.yaml new file mode 100755 index 0000000..4a7e57d --- /dev/null +++ b/test/functest/simulators/simulated-config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: https://aaf-sms.onap:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5
\ No newline at end of file diff --git a/test/optengine-tests/test_modelapi_invalid.json b/test/optengine-tests/test_modelapi_invalid.json new file mode 100644 index 0000000..a58258e --- /dev/null +++ b/test/optengine-tests/test_modelapi_invalid.json @@ -0,0 +1,13 @@ +{ + "requestInfo": { + "transactinId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt" + }, + "modelInfo": { + "modelId": "model2", + "solver": "mzn", + "description": "graph coloring problem for australia", + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_modelapi_valid.json b/test/optengine-tests/test_modelapi_valid.json new file mode 100644 index 0000000..1fbca5b --- /dev/null +++ b/test/optengine-tests/test_modelapi_valid.json @@ -0,0 +1,13 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt" + }, + "modelInfo": { + "modelId": "model2", + "solver": "mzn", + "description": "graph coloring problem for australia", + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid.json b/test/optengine-tests/test_optengine_invalid.json new file mode 100644 index 0000000..9a0267a --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid.json @@ -0,0 +1,18 @@ +{ + "requestInfo": { + "transactioId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];", + "optData": { + "nc": 3 + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid2.json b/test/optengine-tests/test_optengine_invalid2.json new file mode 100644 index 0000000..23c5a8e --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid2.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + + "solverArgs": { + "solver": "cbc" + }, + "modelContent": "% Baking cakes for the school fete (with data file)\r\n\r\nint: flour; %no. grams of flour available\r\nint: banana; %no. of bananas available\r\nint: sugar; %no. grams of sugar available\r\nint: butter; %no. grams of butter available\r\nint: cocoa; %no. grams of cocoa available\r\n\r\nconstraint assert(flour >= 0,\"Invalid datafile: \" ++\r\n \"Amount of flour should be non-negative\");\r\nconstraint assert(banana >= 0,\"Invalid datafile: \" ++\r\n \"Amount of banana should be non-negative\");\r\nconstraint assert(sugar >= 0,\"Invalid datafile: \" ++\r\n \"Amount of sugar should be non-negative\");\r\nconstraint assert(butter >= 0,\"Invalid datafile: \" ++\r\n \"Amount of butter should be non-negative\");\r\nconstraint assert(cocoa >= 0,\"Invalid datafile: \" ++\r\n \"Amount of cocoa should be non-negative\");\r\n\r\nvar 0..100: b; % no. of banana cakes\r\nvar 0..100: c; % no. of chocolate cakes\r\n\r\n% flour\r\nconstraint 250*b + 200*c <= flour;\r\n% bananas\r\nconstraint 2*b <= banana;\r\n% sugar\r\nconstraint 75*b + 150*c <= sugar;\r\n% butter\r\nconstraint 100*b + 150*c <= butter;\r\n% cocoa\r\nconstraint 75*c <= cocoa;\r\n\r\n% maximize our profit\r\nsolve maximize 400*b + 450*c;\r\n\r\noutput [\"no. of banana cakes = \\(b)\\n\",\r\n \"no. of chocolate cakes = \\(c)\\n\"];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid_solver.json b/test/optengine-tests/test_optengine_invalid_solver.json new file mode 100644 index 0000000..a967c16 --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid_solver.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "apy", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_modelId.json b/test/optengine-tests/test_optengine_modelId.json new file mode 100644 index 0000000..b676d91 --- /dev/null +++ b/test/optengine-tests/test_optengine_modelId.json @@ -0,0 +1,19 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "modelId": "test", + "solverArgs": { + "solver": "geocode" + }, + "optData": { + "json": { + "nc": 3 + } + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_no_modelid.json b/test/optengine-tests/test_optengine_no_modelid.json new file mode 100644 index 0000000..9a8c3a4 --- /dev/null +++ b/test/optengine-tests/test_optengine_no_modelid.json @@ -0,0 +1,24 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "cbc" + }, + "modelContent": "% Baking cakes for the school fete (with data file)\r\n\r\nint: flour; %no. grams of flour available\r\nint: banana; %no. of bananas available\r\nint: sugar; %no. grams of sugar available\r\nint: butter; %no. grams of butter available\r\nint: cocoa; %no. grams of cocoa available\r\n\r\nconstraint assert(flour >= 0,\"Invalid datafile: \" ++\r\n \"Amount of flour should be non-negative\");\r\nconstraint assert(banana >= 0,\"Invalid datafile: \" ++\r\n \"Amount of banana should be non-negative\");\r\nconstraint assert(sugar >= 0,\"Invalid datafile: \" ++\r\n \"Amount of sugar should be non-negative\");\r\nconstraint assert(butter >= 0,\"Invalid datafile: \" ++\r\n \"Amount of butter should be non-negative\");\r\nconstraint assert(cocoa >= 0,\"Invalid datafile: \" ++\r\n \"Amount of cocoa should be non-negative\");\r\n\r\nvar 0..100: b; % no. of banana cakes\r\nvar 0..100: c; % no. of chocolate cakes\r\n\r\n% flour\r\nconstraint 250*b + 200*c <= flour;\r\n% bananas\r\nconstraint 2*b <= banana;\r\n% sugar\r\nconstraint 75*b + 150*c <= sugar;\r\n% butter\r\nconstraint 100*b + 150*c <= butter;\r\n% cocoa\r\nconstraint 75*c <= cocoa;\r\n\r\n% maximize our profit\r\nsolve maximize 400*b + 450*c;\r\n\r\noutput [\"no. of banana cakes = \\(b)\\n\",\r\n \"no. of chocolate cakes = \\(c)\\n\"];", + "optData": { + "json": { + "flour": 4000, + "banana": 6, + "sugar": 2000, + "butter": 500, + "cocoa": 500 + } + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_no_optdata.json b/test/optengine-tests/test_optengine_no_optdata.json new file mode 100644 index 0000000..f6645c8 --- /dev/null +++ b/test/optengine-tests/test_optengine_no_optdata.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_solverid.json b/test/optengine-tests/test_optengine_solverid.json new file mode 100644 index 0000000..bfd446c --- /dev/null +++ b/test/optengine-tests/test_optengine_solverid.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "py", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_optengine_valid.json b/test/optengine-tests/test_optengine_valid.json new file mode 100644 index 0000000..8de2b71 --- /dev/null +++ b/test/optengine-tests/test_optengine_valid.json @@ -0,0 +1,20 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];", + "optData": { + "json": { + "nc": 3 + } + } + } +}
\ No newline at end of file diff --git a/test/optengine-tests/test_py_optengine_valid.json b/test/optengine-tests/test_py_optengine_valid.json new file mode 100644 index 0000000..bfd446c --- /dev/null +++ b/test/optengine-tests/test_py_optengine_valid.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "py", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +}
\ No newline at end of file diff --git a/test/test_model_api.py b/test/test_model_api.py new file mode 100644 index 0000000..2a1cecf --- /dev/null +++ b/test/test_model_api.py @@ -0,0 +1,71 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import os + +import pytest +from mock import patch +from schematics.exceptions import DataError + +from runtime.model_api import create_model_data, get_model_data, delete_model_data, retrieve_all_models +from runtime.models.api.model_request import OptimModelRequestAPI +from runtime.optim_engine import validate_request + +BASE_DIR = os.path.dirname(__file__) + +ret_val = {'modelId': 'test', 'description': 'desc', 'solver': 'mzn'} + + +class TestModelApi(): + + def test_valid_mapi_request(self): + req_json = json.loads(open("./test/optengine-tests/test_modelapi_valid.json").read()) + + assert OptimModelRequestAPI(req_json).validate() is None + + def test_invalid_mapi_request(self): + req_json = json.loads(open("./test/optengine-tests/test_modelapi_invalid.json").read()) + with pytest.raises(DataError): + validate_request(req_json) + + @patch('runtime.model_api.build_model_dict') + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_create_model(self, config, conn, model_data): + model_data.return_value = ret_val + req_json = json.loads(open("./test/optengine-tests/test_modelapi_valid.json").read()) + + create_model_data(req_json) + + @patch('runtime.model_api.build_model_dict') + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_retrieve_model(self, config, conn, model_data): + model_data.return_value = ret_val + get_model_data('test') + + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_delete_model(self, config, conn): + delete_model_data('test') + + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_retrieve_all_model(self, config, conn): + retrieve_all_models() diff --git a/test/test_optim_engine.py b/test/test_optim_engine.py new file mode 100644 index 0000000..e1756f8 --- /dev/null +++ b/test/test_optim_engine.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import os + +import pytest +from mock import patch +from schematics.exceptions import DataError + +from osdf.operation.exceptions import BusinessException +from runtime.optim_engine import validate_request, process_request + +BASE_DIR = os.path.dirname(__file__) + + +class TestOptimEngine(): + + def test_valid_optim_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_valid.json").read()) + + assert validate_request(req_json) == True + + def test_invalid_optim_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid.json").read()) + with pytest.raises(DataError): + validate_request(req_json) + + def test_invalid_optim_request_without_modelid(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid2.json").read()) + with pytest.raises(BusinessException): + validate_request(req_json) + + def test_invalid_optim_request_no_optdata(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_no_optdata.json").read()) + with pytest.raises(BusinessException): + validate_request(req_json) + + def test_process_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_valid.json").read()) + + res = process_request(req_json) + assert res.status_code == 400 + + def test_py_process_request(self): + req_json = json.loads(open("./test/optengine-tests/test_py_optengine_valid.json").read()) + + res = process_request(req_json) + assert res.status_code == 200 + + def test_invalid_solver(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid_solver.json").read()) + + with pytest.raises(BusinessException): + process_request(req_json) + + @patch('runtime.optim_engine.get_model_data') + def test_process_solverid_request(self, mocker): + req_json = json.loads(open("./test/optengine-tests/test_optengine_modelId.json").read()) + + data = 200, ('junk', '', '', 'py') + mocker.return_value = data + process_request(req_json) @@ -17,13 +17,14 @@ commands = # TODO: need to update the above "omit" when we package osdf as pip-installable deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test/test-requirements.txt + -r{toxinidir}/requirements-opteng.txt [run] -source=./apps/,./osdf/,osdfapp.py +source=./apps/,./osdf/,osdfapp.py,./runtime/,solverapp.py [testenv:pylint] whitelist_externals=bash -commands = bash -c "pylint --reports=y osdf apps| tee pylint.out" +commands = bash -c "pylint --reports=y osdf apps runtime| tee pylint.out" [testenv:py3] basepython=python3.6 |