From de5fdaafad9fccba0b9a7f308d72f26816dd1a0f Mon Sep 17 00:00:00 2001 From: vrvarma Date: Tue, 3 Mar 2020 22:22:28 -0500 Subject: Adding the generic solver code Add docker file for optim engine Run pods as a non-root user Fix docker tag script Change-Id: If25fe66b839a70e83e35292031a2da012e81fe47 Signed-off-by: vrvarma Issue-ID: OPTFRA-712 --- .coveragerc | 2 +- config/opteng_config.yaml | 25 +++ config/preload_secrets.yaml | 4 + docker/Dockerfile | 69 ------- docker/assembly/osdf-files.xml | 66 ------- docker/build_image.sh | 105 ---------- docker/opteng/Dockerfile | 74 +++++++ docker/opteng/assembly/osdf-files.xml | 55 ++++++ docker/osdf/Dockerfile | 76 ++++++++ docker/osdf/assembly/osdf-files.xml | 66 +++++++ docker/osdf/build_image.sh | 105 ++++++++++ osdf/__init__.py | 5 +- osdf/adapters/aaf/sms.py | 2 + osdf/apps/baseapp.py | 6 +- osdf/utils/file_utils.py | 34 ++++ osdf/utils/mdc_utils.py | 9 +- osdf/webapp/appcontroller.py | 17 +- osdfapp.sh | 45 ++++- pom.xml | 174 ++++++++++------- requirements-opteng.txt | 1 + runtime/__init__.py | 17 ++ runtime/model_api.py | 215 +++++++++++++++++++++ runtime/models/__init__.py | 17 ++ runtime/models/api/__init__.py | 17 ++ runtime/models/api/model_request.py | 48 +++++ runtime/models/api/model_response.py | 31 +++ runtime/models/api/optim_request.py | 60 ++++++ runtime/models/api/optim_response.py | 30 +++ runtime/optim_engine.py | 79 ++++++++ runtime/solvers/__init__.py | 17 ++ runtime/solvers/mzn/__init__.py | 17 ++ runtime/solvers/mzn/mzn_solver.py | 102 ++++++++++ runtime/solvers/py/__init__.py | 17 ++ runtime/solvers/py/py_solver.py | 92 +++++++++ script/TagVersion.groovy | 2 +- solverapp.py | 81 ++++++++ test/config/opteng_config.yaml | 25 +++ .../simulators/simulated-config/opteng_config.yaml | 25 +++ test/optengine-tests/test_modelapi_invalid.json | 13 ++ test/optengine-tests/test_modelapi_valid.json | 13 ++ test/optengine-tests/test_optengine_invalid.json | 18 ++ test/optengine-tests/test_optengine_invalid2.json | 15 ++ .../test_optengine_invalid_solver.json | 15 ++ test/optengine-tests/test_optengine_modelId.json | 19 ++ .../optengine-tests/test_optengine_no_modelid.json | 24 +++ .../optengine-tests/test_optengine_no_optdata.json | 15 ++ test/optengine-tests/test_optengine_solverid.json | 15 ++ test/optengine-tests/test_optengine_valid.json | 20 ++ test/optengine-tests/test_py_optengine_valid.json | 15 ++ test/test_model_api.py | 71 +++++++ test/test_optim_engine.py | 78 ++++++++ tox.ini | 5 +- 52 files changed, 1835 insertions(+), 333 deletions(-) create mode 100755 config/opteng_config.yaml delete mode 100644 docker/Dockerfile delete mode 100644 docker/assembly/osdf-files.xml delete mode 100755 docker/build_image.sh create mode 100644 docker/opteng/Dockerfile create mode 100644 docker/opteng/assembly/osdf-files.xml create mode 100644 docker/osdf/Dockerfile create mode 100644 docker/osdf/assembly/osdf-files.xml create mode 100755 docker/osdf/build_image.sh create mode 100644 osdf/utils/file_utils.py create mode 100644 requirements-opteng.txt create mode 100644 runtime/__init__.py create mode 100644 runtime/model_api.py create mode 100644 runtime/models/__init__.py create mode 100644 runtime/models/api/__init__.py create mode 100644 runtime/models/api/model_request.py create mode 100644 runtime/models/api/model_response.py create mode 100644 runtime/models/api/optim_request.py create mode 100644 runtime/models/api/optim_response.py create mode 100644 runtime/optim_engine.py create mode 100644 runtime/solvers/__init__.py create mode 100644 runtime/solvers/mzn/__init__.py create mode 100644 runtime/solvers/mzn/mzn_solver.py create mode 100644 runtime/solvers/py/__init__.py create mode 100644 runtime/solvers/py/py_solver.py create mode 100644 solverapp.py create mode 100755 test/config/opteng_config.yaml create mode 100755 test/functest/simulators/simulated-config/opteng_config.yaml create mode 100644 test/optengine-tests/test_modelapi_invalid.json create mode 100644 test/optengine-tests/test_modelapi_valid.json create mode 100644 test/optengine-tests/test_optengine_invalid.json create mode 100644 test/optengine-tests/test_optengine_invalid2.json create mode 100644 test/optengine-tests/test_optengine_invalid_solver.json create mode 100644 test/optengine-tests/test_optengine_modelId.json create mode 100644 test/optengine-tests/test_optengine_no_modelid.json create mode 100644 test/optengine-tests/test_optengine_no_optdata.json create mode 100644 test/optengine-tests/test_optengine_solverid.json create mode 100644 test/optengine-tests/test_optengine_valid.json create mode 100644 test/optengine-tests/test_py_optengine_valid.json create mode 100644 test/test_model_api.py create mode 100644 test/test_optim_engine.py diff --git a/.coveragerc b/.coveragerc index a4ec20c..1fa0d3b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,7 +2,7 @@ [run] branch = True cover_pylib = False -include = osdf/**/*.py, apps/**/*.py +include = osdf/**/*.py, apps/**/*.py, runtime/*.py, runtime/**/*.py [report] # Regexes for lines to exclude from consideration diff --git a/config/opteng_config.yaml b/config/opteng_config.yaml new file mode 100755 index 0000000..d6be7ed --- /dev/null +++ b/config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: http://localhost:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5 \ No newline at end of file diff --git a/config/preload_secrets.yaml b/config/preload_secrets.yaml index 0bb2395..b95f1c1 100755 --- a/config/preload_secrets.yaml +++ b/config/preload_secrets.yaml @@ -49,3 +49,7 @@ secrets: values: UserName: pci_test Password: fbf4dcb7f7cda8fdfb742838b0c90ae5bea249801f3f725fdc98941a6e4c347c + - name: osdfOptEngine + values: + UserName: opt_test + Password: 02946408ce6353d45540cd01d912686f19f48c3d8a955d5effdc14c6a43477e5 diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index e339ea7..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# -# ------------------------------------------------------------------------- -# Copyright (c) 2018 AT&T Intellectual Property -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ------------------------------------------------------------------------- -# - -FROM ubuntu:16.04 - -ARG MVN_ARTIFACT_VERSION -ARG REPO -ARG HTTP_PROXY=${HTTP_PROXY} -ARG HTTPS_PROXY=${HTTPS_PROXY} - -ENV http_proxy $HTTP_PROXY -ENV https_proxy $HTTPS_PROXY - -ENV OSDF_PORT "8699" -EXPOSE ${OSDF_PORT} - -ENV MZN 2.4.2 -ENV MZN_BASENAME MiniZincIDE-${MZN}-bundle-linux -ENV MZN_GH_BASE https://github.com/MiniZinc/MiniZincIDE -ENV MZN_DL_URL ${MZN_GH_BASE}/releases/download/${MZN}/${MZN_BASENAME}-x86_64.tgz - -RUN apt-get update -y \ - && apt-get install -y vim unzip wget libmpfr-dev \ - && apt-get install -y git libqt5printsupport5 build-essential \ - && apt-get install -y python3 python3-setuptools python3-dev \ - && easy_install3 pip \ - && pip install --upgrade virtualenv pip wheel setuptools - -RUN ln -s /usr/bin/python3.5 /usr/bin/python - -# Minizinc -RUN wget -q $MZN_DL_URL -O /tmp/mz.tgz \ - && tar xzf /tmp/mz.tgz \ - && mv $MZN_BASENAME /mz-dist \ - && rm /tmp/mz.tgz \ - && echo PATH=/mz-dist/bin:$PATH >> ~/.bashrc - -ENV SHELL /bin/bash -ENV PATH /mz-dist:$PATH - -# OSDF -WORKDIR /opt/osdf -#RUN wget -O /opt/osdf.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=releases&g=org.onap.optf.osdf&a=optf-osdf&e=zip&v=1.3.4" && \ -# unzip -q -o -B /opt/osdf.zip -d /opt/ && \ -# rm -f /opt/osdf.zip - -COPY onap-osdf-tm/optf-osdf-${MVN_ARTIFACT_VERSION}.zip /tmp/optf-osdf.zip -COPY onap-osdf-tm/apps /opt/osdf/apps -RUN unzip -q -o -B /tmp/optf-osdf.zip -d /opt/ && rm -f /tmp/optf-osdf.zip -RUN mkdir -p /var/log/onap/optf/osdf/ -RUN pip install --no-cache-dir -r requirements.txt - -CMD [ "/opt/osdf/osdfapp.sh" ] diff --git a/docker/assembly/osdf-files.xml b/docker/assembly/osdf-files.xml deleted file mode 100644 index fc8a864..0000000 --- a/docker/assembly/osdf-files.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - osdf-files - - - tar.gz - - false - - - - - - ${project.build.finalName}.zip - - ${project.build.directory} - / - - - - apps/** - - - **/*.pyc - **/__pycache__/** - - / - - - - diff --git a/docker/build_image.sh b/docker/build_image.sh deleted file mode 100755 index 3d9d823..0000000 --- a/docker/build_image.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash - -# The script starts in the root folder of the repo, which has the following outline -# We fetch the version information from version.properties, build docker files and -# do a docker push. Since the job will be run under Jenkins, it will have the Nexus -# credentials -# -# ├── docker -# │   ├── Dockerfile -# │   └── build_image.sh <--- THIS SCRIPT is here -# ├── docs -# ├── osdf -# ├── pom.xml -# ├── test -# └── version.properties <--- Version information here - -set -e - -# Folder settings -DOCKER_REPOSITORY=nexus3.onap.org:10003 -ORG=onap -PROJECT=optf-osdf -IMAGE_NAME=$DOCKER_REPOSITORY/$ORG/$PROJECT - -# Version properties -source version.properties -VERSION=$release_version -SNAPSHOT=$snapshot_version -STAGING=${release_version}-STAGING -TIMESTAMP=$(date +"%Y%m%dT%H%M%S")Z -REPO="" - -BUILD_ARGS="--no-cache" -if [ $HTTP_PROXY ]; then - BUILD_ARGS+=" --build-arg HTTP_PROXY=${HTTP_PROXY}" -fi -if [ $HTTPS_PROXY ]; then - BUILD_ARGS+=" --build-arg HTTPS_PROXY=${HTTPS_PROXY}" -fi - -function log_ts() { # Log message with timestamp - echo [DEBUG LOG at $(date -u +%Y%m%d:%H%M%S)] "$@" -} - -function get_artifact_version() { - log_ts Get Maven Artifact version from pom.xml - MVN_ARTIFACT_VERSION=`echo -e "setns x=http://maven.apache.org/POM/4.0.0 \n xpath /x:project/x:version/text() "| xmllint --shell pom.xml | grep content | sed 's/.*content=//'` - log_ts Maven artifact version for OSDF is $MVN_ARTIFACT_VERSION - if [[ "$MVN_ARTIFACT_VERSION" =~ SNAPSHOT ]]; then - log_ts "REPO is snapshots"; - REPO=snapshots - else - log_ts "REPO is releases"; - REPO=releases - fi - BUILD_ARGS+=" --build-arg REPO=${REPO}" - BUILD_ARGS+=" --build-arg MVN_ARTIFACT_VERSION=${MVN_ARTIFACT_VERSION}" -} - -function build_image() { - log_ts Building Image in folder: $PWD with build arguments ${BUILD_ARGS} - docker build ${BUILD_ARGS} -t ${IMAGE_NAME}:latest . - log_ts ... Built -} - -function push_image() { - if [[ "$REPO" == snapshots ]]; then - push_snapshot_image - else - push_staging_image - fi -} - - -function push_snapshot_image(){ - log_ts Tagging images: ${IMAGE_NAME}:\{${SNAPSHOT}-${TIMESTAMP},${SNAPSHOT}-latest\} - docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${SNAPSHOT}-${TIMESTAMP} - docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${SNAPSHOT}-latest - log_ts ... Tagged images - - log_ts Pushing images: ${IMAGE_NAME}:\{${SNAPSHOT}-${TIMESTAMP},${SNAPSHOT}-latest\} - docker push ${IMAGE_NAME}:${SNAPSHOT}-${TIMESTAMP} - docker push ${IMAGE_NAME}:${SNAPSHOT}-latest - log_ts ... Pushed images -} - -function push_staging_image(){ - log_ts Tagging images: ${IMAGE_NAME}:\{${STAGING}-${TIMESTAMP},${STAGING}-latest\} - docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${STAGING}-${TIMESTAMP} - docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${STAGING}-latest - log_ts ... Tagged images - - log_ts Pushing images: ${IMAGE_NAME}:\{${STAGING}-${TIMESTAMP},${STAGING}-latest\} - docker push ${IMAGE_NAME}:${STAGING}-${TIMESTAMP} - docker push ${IMAGE_NAME}:${STAGING}-latest - log_ts ... Pushed images -} - -( - get_artifact_version - # Switch to docker build directory - cd $(dirname $0) - build_image - push_image -) diff --git a/docker/opteng/Dockerfile b/docker/opteng/Dockerfile new file mode 100644 index 0000000..9dca3e7 --- /dev/null +++ b/docker/opteng/Dockerfile @@ -0,0 +1,74 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +FROM python:3.8-alpine + +ARG MVN_ARTIFACT_VERSION +ARG REPO +ARG HTTP_PROXY=${HTTP_PROXY} +ARG HTTPS_PROXY=${HTTPS_PROXY} + +ENV http_proxy $HTTP_PROXY +ENV https_proxy $HTTPS_PROXY + +ENV OSDF_PORT "8699" +EXPOSE ${OSDF_PORT} + +ENV MZN 2.4.2 +ENV MZN_BASENAME MiniZincIDE-${MZN}-bundle-linux +ENV MZN_GH_BASE https://github.com/MiniZinc/MiniZincIDE +ENV MZN_DL_URL ${MZN_GH_BASE}/releases/download/${MZN}/${MZN_BASENAME}-x86_64.tgz + +RUN apk update && apk upgrade \ + && apk --no-cache --update add --virtual build-deps openssl wget \ + && apk --no-cache --update add less ca-certificates bash libxslt-dev unzip \ + freetype freetype-dev libstdc++ build-base libc6-compat \ + && ln -s /lib/libc.musl-x86_64.so.1 /lib/ld-linux-x86-64.so.2 + +# Minizinc +RUN wget -q $MZN_DL_URL -O mz.tgz \ + && tar xzf mz.tgz \ + && mv $MZN_BASENAME /mz-dist \ + && rm mz.tgz \ + && echo PATH=/mz-dist/bin:$PATH >> ~/.bashrc + +ENV SHELL /bin/bash +ENV PATH /mz-dist:$PATH + +RUN addgroup -S onap && adduser -S -G onap onap + +# OSDF +WORKDIR /opt/osdf +#RUN wget -O /opt/osdf.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=releases&g=org.onap.optf.osdf&a=optf-osdf&e=zip&v=1.3.4" && \ +# unzip -q -o -B /opt/osdf.zip -d /opt/ && \ +# rm -f /opt/osdf.zip + +COPY onap-osdf-tm/optf-osdf-${MVN_ARTIFACT_VERSION}.zip /tmp/optf-osdf.zip +COPY onap-osdf-tm/runtime /opt/osdf/runtime +COPY onap-osdf-tm/requirements-opteng.txt . +RUN unzip -q -o -B /tmp/optf-osdf.zip -d /opt/ && rm -f /tmp/optf-osdf.zip +RUN mkdir -p /var/log/onap/optf/osdf/ \ + && chown onap:onap /var/log/onap -R \ + && chown onap:onap /opt/osdf -R + +RUN pip install --no-cache-dir -r requirements.txt -r requirements-opteng.txt + +USER onap + +CMD [ "/opt/osdf/osdfapp.sh", "-x", "solverapp.py", "-c", "/opt/osdf/config/opteng_config.yaml" ] diff --git a/docker/opteng/assembly/osdf-files.xml b/docker/opteng/assembly/osdf-files.xml new file mode 100644 index 0000000..60dd6cc --- /dev/null +++ b/docker/opteng/assembly/osdf-files.xml @@ -0,0 +1,55 @@ + + + osdf-files + + + tar.gz + + false + + + + + + ${project.build.finalName}.zip + + ${project.build.directory} + / + + + + runtime/** + + + **/*.pyc + **/__pycache__/** + + / + + + + requirements-opteng.txt + + / + + + + diff --git a/docker/osdf/Dockerfile b/docker/osdf/Dockerfile new file mode 100644 index 0000000..5860df2 --- /dev/null +++ b/docker/osdf/Dockerfile @@ -0,0 +1,76 @@ +# +# ------------------------------------------------------------------------- +# Copyright (c) 2018 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +FROM ubuntu:16.04 + +ARG MVN_ARTIFACT_VERSION +ARG REPO +ARG HTTP_PROXY=${HTTP_PROXY} +ARG HTTPS_PROXY=${HTTPS_PROXY} + +ENV http_proxy $HTTP_PROXY +ENV https_proxy $HTTPS_PROXY + +ENV OSDF_PORT "8699" +EXPOSE ${OSDF_PORT} + +ENV MZN 2.4.2 +ENV MZN_BASENAME MiniZincIDE-${MZN}-bundle-linux +ENV MZN_GH_BASE https://github.com/MiniZinc/MiniZincIDE +ENV MZN_DL_URL ${MZN_GH_BASE}/releases/download/${MZN}/${MZN_BASENAME}-x86_64.tgz + +RUN apt-get update -y \ + && apt-get install -y vim unzip wget libmpfr-dev \ + && apt-get install -y git libqt5printsupport5 build-essential \ + && apt-get install -y python3 python3-setuptools python3-dev \ + && easy_install3 pip \ + && pip install --upgrade virtualenv pip wheel setuptools + +RUN ln -s /usr/bin/python3.5 /usr/bin/python + +# Minizinc +RUN wget -q $MZN_DL_URL -O /tmp/mz.tgz \ + && tar xzf /tmp/mz.tgz \ + && mv $MZN_BASENAME /mz-dist \ + && rm /tmp/mz.tgz \ + && echo PATH=/mz-dist/bin:$PATH >> ~/.bashrc + +ENV SHELL /bin/bash +ENV PATH /mz-dist:$PATH + +# OSDF +WORKDIR /opt/osdf +#RUN wget -O /opt/osdf.zip "https://nexus.onap.org/service/local/artifact/maven/redirect?r=releases&g=org.onap.optf.osdf&a=optf-osdf&e=zip&v=1.3.4" && \ +# unzip -q -o -B /opt/osdf.zip -d /opt/ && \ +# rm -f /opt/osdf.zip +RUN groupadd onap \ + && useradd -m -g onap onap + +COPY onap-osdf-tm/optf-osdf-${MVN_ARTIFACT_VERSION}.zip /tmp/optf-osdf.zip +COPY onap-osdf-tm/apps /opt/osdf/apps +RUN unzip -q -o -B /tmp/optf-osdf.zip -d /opt/ && rm -f /tmp/optf-osdf.zip +RUN mkdir -p /var/log/onap/optf/osdf/ \ + && chown -R onap:onap /var/log/onap \ + && chown -R onap:onap /opt/osdf + +RUN pip install --no-cache-dir -r requirements.txt + +USER onap + +CMD [ "/opt/osdf/osdfapp.sh", "-x", "osdfapp.py" ] diff --git a/docker/osdf/assembly/osdf-files.xml b/docker/osdf/assembly/osdf-files.xml new file mode 100644 index 0000000..fc8a864 --- /dev/null +++ b/docker/osdf/assembly/osdf-files.xml @@ -0,0 +1,66 @@ + + + + + + osdf-files + + + tar.gz + + false + + + + + + ${project.build.finalName}.zip + + ${project.build.directory} + / + + + + apps/** + + + **/*.pyc + **/__pycache__/** + + / + + + + diff --git a/docker/osdf/build_image.sh b/docker/osdf/build_image.sh new file mode 100755 index 0000000..3d9d823 --- /dev/null +++ b/docker/osdf/build_image.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# The script starts in the root folder of the repo, which has the following outline +# We fetch the version information from version.properties, build docker files and +# do a docker push. Since the job will be run under Jenkins, it will have the Nexus +# credentials +# +# ├── docker +# │   ├── Dockerfile +# │   └── build_image.sh <--- THIS SCRIPT is here +# ├── docs +# ├── osdf +# ├── pom.xml +# ├── test +# └── version.properties <--- Version information here + +set -e + +# Folder settings +DOCKER_REPOSITORY=nexus3.onap.org:10003 +ORG=onap +PROJECT=optf-osdf +IMAGE_NAME=$DOCKER_REPOSITORY/$ORG/$PROJECT + +# Version properties +source version.properties +VERSION=$release_version +SNAPSHOT=$snapshot_version +STAGING=${release_version}-STAGING +TIMESTAMP=$(date +"%Y%m%dT%H%M%S")Z +REPO="" + +BUILD_ARGS="--no-cache" +if [ $HTTP_PROXY ]; then + BUILD_ARGS+=" --build-arg HTTP_PROXY=${HTTP_PROXY}" +fi +if [ $HTTPS_PROXY ]; then + BUILD_ARGS+=" --build-arg HTTPS_PROXY=${HTTPS_PROXY}" +fi + +function log_ts() { # Log message with timestamp + echo [DEBUG LOG at $(date -u +%Y%m%d:%H%M%S)] "$@" +} + +function get_artifact_version() { + log_ts Get Maven Artifact version from pom.xml + MVN_ARTIFACT_VERSION=`echo -e "setns x=http://maven.apache.org/POM/4.0.0 \n xpath /x:project/x:version/text() "| xmllint --shell pom.xml | grep content | sed 's/.*content=//'` + log_ts Maven artifact version for OSDF is $MVN_ARTIFACT_VERSION + if [[ "$MVN_ARTIFACT_VERSION" =~ SNAPSHOT ]]; then + log_ts "REPO is snapshots"; + REPO=snapshots + else + log_ts "REPO is releases"; + REPO=releases + fi + BUILD_ARGS+=" --build-arg REPO=${REPO}" + BUILD_ARGS+=" --build-arg MVN_ARTIFACT_VERSION=${MVN_ARTIFACT_VERSION}" +} + +function build_image() { + log_ts Building Image in folder: $PWD with build arguments ${BUILD_ARGS} + docker build ${BUILD_ARGS} -t ${IMAGE_NAME}:latest . + log_ts ... Built +} + +function push_image() { + if [[ "$REPO" == snapshots ]]; then + push_snapshot_image + else + push_staging_image + fi +} + + +function push_snapshot_image(){ + log_ts Tagging images: ${IMAGE_NAME}:\{${SNAPSHOT}-${TIMESTAMP},${SNAPSHOT}-latest\} + docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${SNAPSHOT}-${TIMESTAMP} + docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${SNAPSHOT}-latest + log_ts ... Tagged images + + log_ts Pushing images: ${IMAGE_NAME}:\{${SNAPSHOT}-${TIMESTAMP},${SNAPSHOT}-latest\} + docker push ${IMAGE_NAME}:${SNAPSHOT}-${TIMESTAMP} + docker push ${IMAGE_NAME}:${SNAPSHOT}-latest + log_ts ... Pushed images +} + +function push_staging_image(){ + log_ts Tagging images: ${IMAGE_NAME}:\{${STAGING}-${TIMESTAMP},${STAGING}-latest\} + docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${STAGING}-${TIMESTAMP} + docker tag ${IMAGE_NAME}:latest ${IMAGE_NAME}:${STAGING}-latest + log_ts ... Tagged images + + log_ts Pushing images: ${IMAGE_NAME}:\{${STAGING}-${TIMESTAMP},${STAGING}-latest\} + docker push ${IMAGE_NAME}:${STAGING}-${TIMESTAMP} + docker push ${IMAGE_NAME}:${STAGING}-latest + log_ts ... Pushed images +} + +( + get_artifact_version + # Switch to docker build directory + cd $(dirname $0) + build_image + push_image +) diff --git a/osdf/__init__.py b/osdf/__init__.py index c33639e..8036d89 100755 --- a/osdf/__init__.py +++ b/osdf/__init__.py @@ -20,11 +20,12 @@ from jinja2 import Template - end_point_auth_mapping = { # map a URL endpoint to auth group "cmscheduler": "CMScheduler", "placement": "Placement", - "pci": "PCIOpt" + "pci": "PCIOpt", + "optmodel": "OptEngine", + "optengine": "OptEngine" } userid_suffix, passwd_suffix = "Username", "Password" diff --git a/osdf/adapters/aaf/sms.py b/osdf/adapters/aaf/sms.py index fd3a5d5..0168ba0 100644 --- a/osdf/adapters/aaf/sms.py +++ b/osdf/adapters/aaf/sms.py @@ -100,6 +100,8 @@ def load_secrets(): config['pciHMSPassword'] = decrypt_pass(secret_dict['pciHMS']['Password']) config['osdfPCIOptUsername'] = secret_dict['osdfPCIOpt']['UserName'] config['osdfPCIOptPassword'] = decrypt_pass(secret_dict['osdfPCIOpt']['Password']) + config['osdfOptEngineUsername'] = secret_dict['osdfOptEngine']['UserName'] + config['osdfOptEnginePassword'] = decrypt_pass(secret_dict['osdfOptEngine']['Password']) cfg_base.http_basic_auth_credentials = creds.load_credentials(osdf_config) cfg_base.dmaap_creds = creds.dmaap_creds() diff --git a/osdf/apps/baseapp.py b/osdf/apps/baseapp.py index 008ce1d..fd94c11 100644 --- a/osdf/apps/baseapp.py +++ b/osdf/apps/baseapp.py @@ -35,7 +35,7 @@ from osdf.config.base import osdf_config from osdf.logging.osdf_logging import error_log, debug_log from osdf.operation.error_handling import request_exception_to_json_body, internal_error_message from osdf.operation.exceptions import BusinessException -from osdf.utils.mdc_utils import clear_mdc, mdc_from_json, default_mdc +from osdf.utils.mdc_utils import clear_mdc, mdc_from_json, default_mdc, get_request_id from requests import RequestException from schematics.exceptions import DataError @@ -88,11 +88,11 @@ def handle_data_error(e): @app.before_request def log_request(): - g.request_start = time.clock() + g.request_start = time.process_time() if request.data: if request.get_json(): request_json = request.get_json() - g.request_id = request_json['requestInfo']['requestId'] + g.request_id = get_request_id(request_json) mdc_from_json(request_json) else: g.request_id = "N/A" diff --git a/osdf/utils/file_utils.py b/osdf/utils/file_utils.py new file mode 100644 index 0000000..b12c17d --- /dev/null +++ b/osdf/utils/file_utils.py @@ -0,0 +1,34 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +# File related utilities + +import os +from shutil import rmtree + +from osdf.logging.osdf_logging import debug_log + + +def delete_file_folder(p): + if not p: + return + debug_log.debug('Deleting folder/file {}'.format(p)) + if os.path.isfile(p): + os.remove(p) + else: + rmtree(p, ignore_errors=True) diff --git a/osdf/utils/mdc_utils.py b/osdf/utils/mdc_utils.py index bcd0615..14b726d 100644 --- a/osdf/utils/mdc_utils.py +++ b/osdf/utils/mdc_utils.py @@ -53,9 +53,16 @@ def default_mdc(): def mdc_from_json(request_json): default_mdc() - MDC.put('requestID', request_json['requestInfo']['requestId']) + MDC.put('requestID', get_request_id(request_json)) MDC.put('partnerName', request_json['requestInfo']['sourceId']) +def get_request_id(request_json): + request_id = request_json['requestInfo'].get('requestId') + if not request_id: + request_id = request_json['requestInfo'].get('requestID') + return request_id + + def clear_mdc(): MDC.clear() diff --git a/osdf/webapp/appcontroller.py b/osdf/webapp/appcontroller.py index e48e93f..5db879a 100644 --- a/osdf/webapp/appcontroller.py +++ b/osdf/webapp/appcontroller.py @@ -16,14 +16,16 @@ # ------------------------------------------------------------------------- # +import json + +from flask import Response from flask import request from flask_httpauth import HTTPBasicAuth -from flask import Response -import json + import osdf import osdf.config.base as cfg_base -from osdf.config.base import osdf_config from osdf.adapters.aaf import aaf_authentication as aaf_auth +from osdf.config.base import osdf_config auth_basic = HTTPBasicAuth() @@ -38,10 +40,11 @@ unauthorized_message = json.dumps(error_body) @auth_basic.get_password def get_pw(username): - end_point = request.url.split('/')[-1] - auth_group = osdf.end_point_auth_mapping.get(end_point) - return cfg_base.http_basic_auth_credentials[auth_group].get( - username) if auth_group else None + auth_group = '' + for k in osdf.end_point_auth_mapping: + if k in request.url: + auth_group = osdf.end_point_auth_mapping.get(k) + return cfg_base.http_basic_auth_credentials[auth_group].get(username) if auth_group else None @auth_basic.error_handler diff --git a/osdfapp.sh b/osdfapp.sh index 25e3c05..3dc4679 100755 --- a/osdfapp.sh +++ b/osdfapp.sh @@ -18,16 +18,46 @@ # ------------------------------------------------------------------------- # +usage() { + echo "Usage:" + echo " $0 -h Display this help message." + echo " $0 -c configfile_path(optional) -x app.py file" + exit 0 +} + cd $(dirname $0) # bash ../etc/make-certs.sh # create the https certificates if they are not present +while getopts ":hc:x:" opt; do + case ${opt} in + h ) + usage + ;; + c ) + # process option configuration + export OSDF_CONFIG_FILE=$OPTARG + ;; + x ) + # process executable file + export EXEC_FILE=$OPTARG + ;; + ? ) + usage + ;; + : ) + echo "Invalid Option: -$OPTARG requires an argument" 1>&2 + exit 1 + ;; + esac +done +shift $(( OPTIND - 1 )) + +set -e + LOGS=logs mkdir -p $LOGS -export OSDF_CONFIG_FILE=${1:-/opt/app/config/osdf_config.yaml} # this file may be passed by invoker -[ ! -e "$OSDF_CONFIG_FILE" ] && unset OSDF_CONFIG_FILE - if [ -e /opt/app/ssl_cert/aaf_root_ca.cer ]; then #assuming that this would be an ubuntu vm. cp /opt/app/ssl_cert/aaf_root_ca.cer /usr/local/share/ca-certificates/aafcacert.crt @@ -41,4 +71,11 @@ else export REQUESTS_CA_BUNDLE=/opt/app/ssl_cert/aaf_root_ca.cer fi -python osdfapp.py 2>$LOGS/err.log 1>$LOGS/out.log < /dev/null # running the app +if [ ! -z "$EXEC_FILE" ] +then + # flask run + echo "Running $EXEC_FILE" + python $EXEC_FILE # running the app +else + usage +fi diff --git a/pom.xml b/pom.xml index 3f15e9f..a3311b1 100644 --- a/pom.xml +++ b/pom.xml @@ -14,77 +14,80 @@ License for the specific language governing permissions and limitations under the License. --> - - 4.0.0 - pom + + 4.0.0 + pom - - org.onap.oparent - oparent-python - 3.0.0 - + + org.onap.oparent + oparent-python + 3.0.0 + - org.onap.optf.osdf - optf-osdf - optf-osdf - 1.3.4-SNAPSHOT - Optimization Service Design Framework + org.onap.optf.osdf + optf-osdf + optf-osdf + 1.3.4-SNAPSHOT + Optimization Service Design Framework - - UTF-8 - . - xunit-results.xml - coverage.xml - py - python - **/**.py,osdfapp.py - test/**.py,docs/**.py - yyyyMMdd'T'HHmmss'Z' - ${maven.build.timestamp} - ${project.version} - nexus3.onap.org:10003 + + UTF-8 + . + xunit-results.xml + coverage.xml + py + python + **/**.py,osdfapp.py + test/**.py,docs/**.py + yyyyMMdd'T'HHmmss'Z' + ${maven.build.timestamp} + ${project.version} + nexus3.onap.org:10003 ${osdf.docker.repository}/onap/optf-osdf - + ${osdf.docker.repository}/onap/optf-opteng + - - - - - exec-maven-plugin - org.codehaus.mojo - - - maven-assembly-plugin - - false - - assembly.xml - - - - - make-assembly - package - - single - - - - + + + + + exec-maven-plugin + org.codehaus.mojo + + + maven-assembly-plugin + + false + + assembly.xml + + + + + make-assembly + package + + single + + + + - - org.apache.maven.plugins - maven-release-plugin - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8 - - 2 - - - + + org.apache.maven.plugins + maven-release-plugin + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8 + + 2 + + + org.codehaus.groovy.maven gmaven-plugin 1.0 @@ -100,7 +103,7 @@ - + io.fabric8 docker-maven-plugin 0.26.0 @@ -119,14 +122,14 @@ ${project.docker.latesttag.version} - ${project.basedir}/docker/Dockerfile + ${project.basedir}/docker/osdf/Dockerfile - ${project.basedir}/docker/assembly/osdf-files.xml + ${project.basedir}/docker/osdf/assembly/osdf-files.xml onap-osdf-tm - ${project.version} - ${project.repo} + ${project.version} + ${project.repo} + + + @@ -158,6 +188,6 @@ - - + + diff --git a/requirements-opteng.txt b/requirements-opteng.txt new file mode 100644 index 0000000..6d0b524 --- /dev/null +++ b/requirements-opteng.txt @@ -0,0 +1 @@ +mysql-connector-python>=8.0.12 \ No newline at end of file diff --git a/runtime/__init__.py b/runtime/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/model_api.py b/runtime/model_api.py new file mode 100644 index 0000000..fd87333 --- /dev/null +++ b/runtime/model_api.py @@ -0,0 +1,215 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import traceback + +import mysql.connector +from flask import g, Flask, Response + +from osdf.config.base import osdf_config +from osdf.logging.osdf_logging import debug_log, error_log +from osdf.operation.exceptions import BusinessException + + +def init_db(): + if is_db_enabled(): + get_db() + + +def get_db(): + """Opens a new database connection if there is none yet for the + current application context. + """ + if not hasattr(g, 'pg'): + properties = osdf_config['deployment'] + host, db_port, db = properties["osdfDatabaseHost"], properties["osdfDatabasePort"], \ + properties.get("osdfDatabaseSchema") + user, password = properties["osdfDatabaseUsername"], properties["osdfDatabasePassword"] + g.pg = mysql.connector.connect(host=host, port=db_port, user=user, password=password, database=db) + return g.pg + + +def close_db(): + """Closes the database again at the end of the request.""" + if hasattr(g, 'pg'): + g.pg.close() + + +app = Flask(__name__) + + +def create_model_data(model_api): + with app.app_context(): + try: + model_info = model_api['modelInfo'] + model_id = model_info['modelId'] + debug_log.debug( + "persisting model_api {}".format(model_id)) + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id FROM optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + if cursor.fetchone() is None: + query = "INSERT INTO optim_model_data (model_id, model_content, description, solver_type) VALUES " \ + "(%s, %s, %s, %s)" + values = (model_id, model_info['modelContent'], model_info.get('description'), model_info['solver']) + cursor.execute(query, values) + g.pg.commit() + + debug_log.debug("A record successfully inserted for request_id: {}".format(model_id)) + return retrieve_model_data(model_id) + close_db() + else: + query = "UPDATE optim_model_data SET model_content = %s, description = %s, solver_type = %s where " \ + "model_id = %s " + values = (model_info['modelContent'], model_info.get('description'), model_info['solver'], model_id) + cursor.execute(query, values) + g.pg.commit() + + return retrieve_model_data(model_id) + close_db() + except Exception as err: + error_log.error("error for request_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + raise BusinessException(err) + + +def retrieve_model_data(model_id): + status, resp_data = get_model_data(model_id) + + if status == 200: + resp = json.dumps(build_model_dict(resp_data)) + return build_response(resp, status) + else: + resp = json.dumps({ + 'modelId': model_id, + 'statusMessage': "Error retrieving the model data for model {} due to {}".format(model_id, resp_data) + }) + return build_response(resp, status) + + +def build_model_dict(resp_data, content_needed=True): + resp = {'modelId': resp_data[0], 'description': resp_data[2] if resp_data[2] else '', + 'solver': resp_data[3]} + if content_needed: + resp.update({'modelContent': resp_data[1]}) + return resp + + +def build_response(resp, status): + response = Response(resp, content_type='application/json; charset=utf-8') + response.headers.add('content-length', len(resp)) + response.status_code = status + return response + + +def delete_model_data(model_id): + with app.app_context(): + try: + debug_log.debug("deleting model data given model_id = {}".format(model_id)) + d = dict(); + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "delete from optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + g.pg.commit() + close_db() + resp = { + "statusMessage": "model data for modelId {} deleted".format(model_id) + } + return build_response(json.dumps(resp), 200) + except Exception as err: + error_log.error("error deleting model_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + raise BusinessException(err) + + +def get_model_data(model_id): + with app.app_context(): + try: + debug_log.debug("getting model data given model_id = {}".format(model_id)) + d = dict(); + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id, model_content, description, solver_type FROM optim_model_data WHERE model_id = %s" + values = (model_id,) + cursor.execute(query, values) + if cursor is None: + return 400, "FAILED" + else: + rows = cursor.fetchone() + if rows is not None: + index = 0 + for row in rows: + d[index] = row + index = index + 1 + return 200, d + else: + close_db() + return 500, "NOT_FOUND" + except Exception: + error_log.error("error for request_id: {} - {}".format(model_id, traceback.format_exc())) + close_db() + return 500, "FAILED" + + +def retrieve_all_models(): + status, resp_data = get_all_models() + model_list = [] + if status == 200: + for r in resp_data: + model_list.append(build_model_dict(r, False)) + resp = json.dumps(model_list) + return build_response(resp, status) + + else: + resp = json.dumps({ + 'statusMessage': "Error retrieving all the model data due to {}".format(resp_data) + }) + return build_response(resp, status) + + +def get_all_models(): + with app.app_context(): + try: + debug_log.debug("getting all model data".format()) + connection = get_db() + cursor = connection.cursor(buffered=True) + query = "SELECT model_id, model_content, description, solver_type FROM optim_model_data" + + cursor.execute(query) + if cursor is None: + return 400, "FAILED" + else: + rows = cursor.fetchall() + if rows is not None: + return 200, rows + else: + close_db() + return 500, "NOT_FOUND" + except Exception: + error_log.error("error for request_id: {}".format(traceback.format_exc())) + close_db() + return 500, "FAILED" + + +def is_db_enabled(): + return osdf_config['deployment'].get('isDatabaseEnabled', False) diff --git a/runtime/models/__init__.py b/runtime/models/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/models/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/models/api/__init__.py b/runtime/models/api/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/models/api/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/models/api/model_request.py b/runtime/models/api/model_request.py new file mode 100644 index 0000000..710da4b --- /dev/null +++ b/runtime/models/api/model_request.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType +from schematics.types.compound import ModelType + +from osdf.models.api.common import OSDFModel + + +class RequestInfo(OSDFModel): + """Info for northbound request from client such as PCI-mS Handler""" + transactionId = StringType(required=True) + requestID = StringType(required=True) + sourceId = StringType(required=True) + + +class OptimModelInfo(OSDFModel): + """Optimizer request info details.""" + # ModelId from the database + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType(required=True) + # Description of the model + description = StringType() + # a large blob string containing the model (which is not that + # problematic since models are fairly small). + modelContent = StringType() + + +class OptimModelRequestAPI(OSDFModel): + """Request for Optimizer API (specific to optimization and additional metadata""" + requestInfo = ModelType(RequestInfo, required=True) + modelInfo = ModelType(OptimModelInfo, required=True) diff --git a/runtime/models/api/model_response.py b/runtime/models/api/model_response.py new file mode 100644 index 0000000..e4a41a5 --- /dev/null +++ b/runtime/models/api/model_response.py @@ -0,0 +1,31 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType + +from osdf.models.api.common import OSDFModel + + +class OptimModelResponse(OSDFModel): + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType() + # a large blob string containing the model + modelContent = StringType() + # statusMessage + statusMessage = StringType() diff --git a/runtime/models/api/optim_request.py b/runtime/models/api/optim_request.py new file mode 100644 index 0000000..4a046d2 --- /dev/null +++ b/runtime/models/api/optim_request.py @@ -0,0 +1,60 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import BaseType, DictType, StringType, IntType +from schematics.types.compound import ModelType + +from osdf.models.api.common import OSDFModel + +""" +""" +class RequestInfo(OSDFModel): + """Info for northbound request from client """ + transactionId = StringType(required=True) + requestID = StringType(required=True) + callbackUrl = StringType() + sourceId = StringType(required=True) + timeout = IntType() + + +class DataInfo(OSDFModel): + """Optimization data info""" + text = StringType() + json = DictType(BaseType) + + +class OptimInfo(OSDFModel): + """Optimizer request info details.""" + # ModelId from the database, if its not populated, + # assume that solverModel will be populated. + modelId = StringType() + # type of solver (mzn, or-tools, etc.) + solver = StringType() + # Arguments for solver + solverArgs = DictType(BaseType) + # NOTE: a large blob string containing the model (which is not that + # problematic since models are fairly small). + modelContent = StringType() + # Data Payload, input data for the solver + optData = ModelType(DataInfo) + + +class OptimizationAPI(OSDFModel): + """Request for Optimizer API (specific to optimization and additional metadata""" + requestInfo = ModelType(RequestInfo, required=True) + optimInfo = ModelType(OptimInfo, required=True) diff --git a/runtime/models/api/optim_response.py b/runtime/models/api/optim_response.py new file mode 100644 index 0000000..6fd0f6b --- /dev/null +++ b/runtime/models/api/optim_response.py @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from schematics.types import StringType, BaseType +from schematics.types.compound import DictType + +from osdf.models.api.common import OSDFModel + + +class OptimResponse(OSDFModel): + transactionId = StringType(required=True) + requestID = StringType(required=True) + requestStatus = StringType(required=True) + statusMessage = StringType() + solutions = DictType(BaseType) diff --git a/runtime/optim_engine.py b/runtime/optim_engine.py new file mode 100644 index 0000000..4a8788e --- /dev/null +++ b/runtime/optim_engine.py @@ -0,0 +1,79 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from flask import Response + +from osdf.operation.exceptions import BusinessException +from .model_api import get_model_data +from .models.api.optim_request import OptimizationAPI +from .solvers.mzn.mzn_solver import solve as mzn_solve +from .solvers.py.py_solver import solve as py_solve + + +def is_valid_optim_request(request_json): + # Method to check whether the requestinfo/optimizer value is valid. + opt_info = request_json['optimInfo'] + if not opt_info.get('modelId'): + if not opt_info.get('modelContent') or not opt_info.get('solver'): + raise BusinessException('modelContent and solver needs to be populated if model_id is not set') + if not opt_info.get('optData'): + raise BusinessException('optimInfo.optData needs to be populated to solve for a problem') + + return True + + +def validate_request(request_json): + OptimizationAPI(request_json).validate() + if not is_valid_optim_request(request_json): + raise BusinessException('Invalid optim request ') + return True + + +def process_request(request_json): + response_code, response_message = run_optimizer(request_json) + response = Response(response_message, content_type='application/json; charset=utf-8') + response.headers.add('content-length', len(response_message)) + response.status_code = response_code + return response + + +def run_optimizer(request_json): + validate_request(request_json) + + model_content, solver = get_model_content(request_json) + + if solver == 'mzn': + return mzn_solve(request_json, model_content) + elif solver == 'py': + return py_solve(request_json, model_content) + raise BusinessException('Unsupported optimization solver requested {} '.format(solver)) + + +def get_model_content(request_json): + model_id = request_json['optimInfo'].get('modelId') + if model_id: + status, data = get_model_data(model_id) + if status == 200: + model_content = data[1] + solver = data[3] + else: + raise BusinessException('model_id [{}] not found in the model database'.format(model_id)) + else: + model_content = request_json['optimInfo']['modelContent'] + solver = request_json['optimInfo']['solver'] + return model_content, solver diff --git a/runtime/solvers/__init__.py b/runtime/solvers/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/solvers/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/solvers/mzn/__init__.py b/runtime/solvers/mzn/__init__.py new file mode 100644 index 0000000..2aa67d8 --- /dev/null +++ b/runtime/solvers/mzn/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# diff --git a/runtime/solvers/mzn/mzn_solver.py b/runtime/solvers/mzn/mzn_solver.py new file mode 100644 index 0000000..cf002e7 --- /dev/null +++ b/runtime/solvers/mzn/mzn_solver.py @@ -0,0 +1,102 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +from datetime import datetime + +from pymzn import Status, minizinc, cbc, gecode, chuffed, or_tools + +from osdf.utils.file_utils import delete_file_folder + +error_status_map = { + Status.INCOMPLETE: "incomplete", + Status.COMPLETE: "complete", + Status.UNSATISFIABLE: "unsatisfiable", + Status.UNKNOWN: "unknown", + Status.UNBOUNDED: "unbounded", + Status.UNSATorUNBOUNDED: "unsat_or_unbounded", + Status.ERROR: "error" +} + +solver_dict = { + 'cbc': cbc, + 'geocode': gecode, + 'chuffed': chuffed, + 'cp': chuffed, + 'or_tools': or_tools +} + + +def map_status(status): + return error_status_map.get(status, "failed") + + +def solve(request_json, mzn_content): + req_info = request_json['requestInfo'] + opt_info = request_json['optimInfo'] + try: + mzn_solution = mzn_solver(mzn_content, opt_info) + + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'done', + 'statusMessage': map_status(mzn_solution.status), + 'solutions': mzn_solution[0] if mzn_solution else {} + } + return 200, json.dumps(response) + except Exception as e: + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'failed', + 'statusMessage': 'Failed due to {}'.format(e) + } + return 400, json.dumps(response) + + +def mzn_solver(mzn_content, opt_info): + args = opt_info['solverArgs'] + solver = get_mzn_solver(args.pop('solver')) + mzn_opts = dict() + + try: + file_name = persist_opt_data(opt_info) + mzn_opts.update(args) + return minizinc(mzn_content, file_name, **mzn_opts, solver=solver) + + finally: + delete_file_folder(file_name) + + +def persist_opt_data(opt_info): + + if opt_info['optData'].get('json'): + data_content = json.dumps(opt_info['optData']['json']) + file_name = '/tmp/optim_engine_{}.json'.format(datetime.timestamp(datetime.now())) + elif opt_info['optData'].get('text'): + data_content = opt_info['optData']['text'] + file_name = '/tmp/optim_engine_{}.dzn'.format(datetime.timestamp(datetime.now())) + + with open(file_name, "wt") as data: + data.write(data_content) + return file_name + + +def get_mzn_solver(solver): + return solver_dict.get(solver) diff --git a/runtime/solvers/py/__init__.py b/runtime/solvers/py/__init__.py new file mode 100644 index 0000000..a8aa582 --- /dev/null +++ b/runtime/solvers/py/__init__.py @@ -0,0 +1,17 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# \ No newline at end of file diff --git a/runtime/solvers/py/py_solver.py b/runtime/solvers/py/py_solver.py new file mode 100644 index 0000000..6b200ab --- /dev/null +++ b/runtime/solvers/py/py_solver.py @@ -0,0 +1,92 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import subprocess +import traceback +from datetime import datetime + +from osdf.logging.osdf_logging import error_log, debug_log +from osdf.utils.file_utils import delete_file_folder + + +def py_solver(py_content, opt_info): + py_file = '/tmp/custom_heuristics_{}.py'.format(datetime.timestamp(datetime.now())) + with open(py_file, "wt") as f: + f.write(py_content) + if opt_info['optData'].get('json'): + data_content = json.dumps(opt_info['optData']['json']) + input_file = '/tmp/optim_engine_{}.json'.format(datetime.timestamp(datetime.now())) + elif opt_info['optData'].get('text'): + data_content = opt_info['optData']['text'] + input_file = '/tmp/optim_engine_{}.txt'.format(datetime.timestamp(datetime.now())) + with open(input_file, "wt") as f: + f.write(data_content) + + output_file = '/tmp/opteng_output_{}.json'.format(datetime.timestamp(datetime.now())) + + command = ['python', py_file, input_file, output_file] + + try: + p = subprocess.run(command, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + + debug_log.debug('Process return code {}'.format(p.returncode)) + if p.returncode > 0: + error_log.error('Process return code {} {}'.format(p.returncode, p.stdout)) + return 'error', {} + with open(output_file) as file: + data = file.read() + return 'success', json.loads(data) + + except Exception as e: + error_log.error("Error running optimizer {}".format(traceback.format_exc())) + return 'error', {} + finally: + cleanup((input_file, output_file, py_file)) + + +def cleanup(file_tup): + for f in file_tup: + try: + delete_file_folder(f) + except Exception as e: + error_log.error("Failed deleting the file {} - {}".format(f, traceback.format_exc())) + + +def solve(request_json, py_content): + req_info = request_json['requestInfo'] + opt_info = request_json['optimInfo'] + try: + status, solution = py_solver(py_content, opt_info) + + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': status, + 'statusMessage': "completed", + 'solutions': solution if solution else {} + } + return 200, json.dumps(response) + except Exception as e: + response = { + 'transactionId': req_info['transactionId'], + 'requestID': req_info['requestID'], + 'requestStatus': 'failed', + 'statusMessage': 'Failed due to {}'.format(e) + } + return 400, json.dumps(response) diff --git a/script/TagVersion.groovy b/script/TagVersion.groovy index 6ed6558..01bc840 100644 --- a/script/TagVersion.groovy +++ b/script/TagVersion.groovy @@ -37,7 +37,7 @@ if ( project.properties['osdf.project.version'].endsWith("-SNAPSHOT") ) { project.properties['project.docker.latesttagtimestamp.version']=versionTag + "-SNAPSHOT-"+timestamp; project.properties['project.repo'] = 'snapshots' } else { - project.properties['project.docker.latesttag.version']=baseTag + "-STAGING-latest"; + project.properties['project.docker.latesttag.version']=versionTag + "-STAGING-latest"; project.properties['project.docker.latesttagtimestamp.version']=versionTag + "-STAGING-"+timestamp; project.properties['project.repo'] = 'releases' } diff --git a/solverapp.py b/solverapp.py new file mode 100644 index 0000000..39f2670 --- /dev/null +++ b/solverapp.py @@ -0,0 +1,81 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +from flask import request, g + +from osdf.apps.baseapp import app, run_app +from osdf.logging.osdf_logging import audit_log +from osdf.webapp.appcontroller import auth_basic +from runtime.model_api import create_model_data, retrieve_model_data, retrieve_all_models, delete_model_data +from runtime.models.api.model_request import OptimModelRequestAPI +from runtime.optim_engine import process_request + + +@app.route("/api/oof/optengine/v1", methods=["POST"]) +@auth_basic.login_required +def opt_engine_rest_api(): + """Perform OptimEngine optimization after validating the request + """ + request_json = request.get_json() + return process_request(request_json) + + +@app.route("/api/oof/optmodel/v1", methods=["PUT", "POST"]) +@auth_basic.login_required +def opt_model_create_rest_api(): + """Perform OptimEngine optimization after validating the request + """ + request_json = request.get_json() + OptimModelRequestAPI(request_json).validate() + return create_model_data(request_json) + + +@app.route("/api/oof/optmodel/v1/", methods=["GET"]) +@auth_basic.login_required +def opt_get_model_rest_api(model_id): + """Retrieve model data + """ + + return retrieve_model_data(model_id) + + +@app.route("/api/oof/optmodel/v1", methods=["GET"]) +@auth_basic.login_required +def opt_get_all_models_rest_api(): + """Retrieve all models data + """ + return retrieve_all_models() + + +@app.route("/api/oof/optmodel/v1/", methods=["DELETE"]) +@auth_basic.login_required +def opt_delete_model_rest_api(model_id): + """Perform OptimEngine optimization after validating the request + """ + return delete_model_data(model_id) + + +@app.route("/api/oof/optengine/healthcheck/v1", methods=["GET"]) +def do_health_check(): + """Simple health check""" + audit_log.info("A OptimEngine health check v1 request is processed!") + return "OK" + + +if __name__ == "__main__": + run_app() diff --git a/test/config/opteng_config.yaml b/test/config/opteng_config.yaml new file mode 100755 index 0000000..4a7e57d --- /dev/null +++ b/test/config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: https://aaf-sms.onap:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5 \ No newline at end of file diff --git a/test/functest/simulators/simulated-config/opteng_config.yaml b/test/functest/simulators/simulated-config/opteng_config.yaml new file mode 100755 index 0000000..4a7e57d --- /dev/null +++ b/test/functest/simulators/simulated-config/opteng_config.yaml @@ -0,0 +1,25 @@ +# Policy Platform -- requires Authorization +policyPlatformUrl: https://policy-xacml-pdp:6969/policy/pdpx/decision/v1 # Policy Dev platform URL + +# AAF Authentication config +is_aaf_enabled: False +aaf_cache_expiry_mins: 5 +aaf_url: https://aaftest.simpledemo.onap.org:8095 +aaf_user_roles: + - '/optmodel:org.onap.oof.access|*|read ALL' + - '/optengine:org.onap.oof.access|*|read ALL' + +# Secret Management Service from AAF +aaf_sms_url: https://aaf-sms.onap:10443 +aaf_sms_timeout: 30 +secret_domain: osdf +aaf_ca_certs: ssl_certs/aaf_root_ca.cer + +osdfDatabaseHost: localhost +osdfDatabaseSchema: osdf +osdfDatabaseUsername: osdf +osdfDatabasePassword: osdf +osdfDatabasePort: 3306 + +#key +appkey: os35@rrtky400fdntc#001t5 \ No newline at end of file diff --git a/test/optengine-tests/test_modelapi_invalid.json b/test/optengine-tests/test_modelapi_invalid.json new file mode 100644 index 0000000..a58258e --- /dev/null +++ b/test/optengine-tests/test_modelapi_invalid.json @@ -0,0 +1,13 @@ +{ + "requestInfo": { + "transactinId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt" + }, + "modelInfo": { + "modelId": "model2", + "solver": "mzn", + "description": "graph coloring problem for australia", + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_modelapi_valid.json b/test/optengine-tests/test_modelapi_valid.json new file mode 100644 index 0000000..1fbca5b --- /dev/null +++ b/test/optengine-tests/test_modelapi_valid.json @@ -0,0 +1,13 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt" + }, + "modelInfo": { + "modelId": "model2", + "solver": "mzn", + "description": "graph coloring problem for australia", + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid.json b/test/optengine-tests/test_optengine_invalid.json new file mode 100644 index 0000000..9a0267a --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid.json @@ -0,0 +1,18 @@ +{ + "requestInfo": { + "transactioId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];", + "optData": { + "nc": 3 + } + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid2.json b/test/optengine-tests/test_optengine_invalid2.json new file mode 100644 index 0000000..23c5a8e --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid2.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + + "solverArgs": { + "solver": "cbc" + }, + "modelContent": "% Baking cakes for the school fete (with data file)\r\n\r\nint: flour; %no. grams of flour available\r\nint: banana; %no. of bananas available\r\nint: sugar; %no. grams of sugar available\r\nint: butter; %no. grams of butter available\r\nint: cocoa; %no. grams of cocoa available\r\n\r\nconstraint assert(flour >= 0,\"Invalid datafile: \" ++\r\n \"Amount of flour should be non-negative\");\r\nconstraint assert(banana >= 0,\"Invalid datafile: \" ++\r\n \"Amount of banana should be non-negative\");\r\nconstraint assert(sugar >= 0,\"Invalid datafile: \" ++\r\n \"Amount of sugar should be non-negative\");\r\nconstraint assert(butter >= 0,\"Invalid datafile: \" ++\r\n \"Amount of butter should be non-negative\");\r\nconstraint assert(cocoa >= 0,\"Invalid datafile: \" ++\r\n \"Amount of cocoa should be non-negative\");\r\n\r\nvar 0..100: b; % no. of banana cakes\r\nvar 0..100: c; % no. of chocolate cakes\r\n\r\n% flour\r\nconstraint 250*b + 200*c <= flour;\r\n% bananas\r\nconstraint 2*b <= banana;\r\n% sugar\r\nconstraint 75*b + 150*c <= sugar;\r\n% butter\r\nconstraint 100*b + 150*c <= butter;\r\n% cocoa\r\nconstraint 75*c <= cocoa;\r\n\r\n% maximize our profit\r\nsolve maximize 400*b + 450*c;\r\n\r\noutput [\"no. of banana cakes = \\(b)\\n\",\r\n \"no. of chocolate cakes = \\(c)\\n\"];" + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_invalid_solver.json b/test/optengine-tests/test_optengine_invalid_solver.json new file mode 100644 index 0000000..a967c16 --- /dev/null +++ b/test/optengine-tests/test_optengine_invalid_solver.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "apy", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_modelId.json b/test/optengine-tests/test_optengine_modelId.json new file mode 100644 index 0000000..b676d91 --- /dev/null +++ b/test/optengine-tests/test_optengine_modelId.json @@ -0,0 +1,19 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "modelId": "test", + "solverArgs": { + "solver": "geocode" + }, + "optData": { + "json": { + "nc": 3 + } + } + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_no_modelid.json b/test/optengine-tests/test_optengine_no_modelid.json new file mode 100644 index 0000000..9a8c3a4 --- /dev/null +++ b/test/optengine-tests/test_optengine_no_modelid.json @@ -0,0 +1,24 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "cbc" + }, + "modelContent": "% Baking cakes for the school fete (with data file)\r\n\r\nint: flour; %no. grams of flour available\r\nint: banana; %no. of bananas available\r\nint: sugar; %no. grams of sugar available\r\nint: butter; %no. grams of butter available\r\nint: cocoa; %no. grams of cocoa available\r\n\r\nconstraint assert(flour >= 0,\"Invalid datafile: \" ++\r\n \"Amount of flour should be non-negative\");\r\nconstraint assert(banana >= 0,\"Invalid datafile: \" ++\r\n \"Amount of banana should be non-negative\");\r\nconstraint assert(sugar >= 0,\"Invalid datafile: \" ++\r\n \"Amount of sugar should be non-negative\");\r\nconstraint assert(butter >= 0,\"Invalid datafile: \" ++\r\n \"Amount of butter should be non-negative\");\r\nconstraint assert(cocoa >= 0,\"Invalid datafile: \" ++\r\n \"Amount of cocoa should be non-negative\");\r\n\r\nvar 0..100: b; % no. of banana cakes\r\nvar 0..100: c; % no. of chocolate cakes\r\n\r\n% flour\r\nconstraint 250*b + 200*c <= flour;\r\n% bananas\r\nconstraint 2*b <= banana;\r\n% sugar\r\nconstraint 75*b + 150*c <= sugar;\r\n% butter\r\nconstraint 100*b + 150*c <= butter;\r\n% cocoa\r\nconstraint 75*c <= cocoa;\r\n\r\n% maximize our profit\r\nsolve maximize 400*b + 450*c;\r\n\r\noutput [\"no. of banana cakes = \\(b)\\n\",\r\n \"no. of chocolate cakes = \\(c)\\n\"];", + "optData": { + "json": { + "flour": 4000, + "banana": 6, + "sugar": 2000, + "butter": 500, + "cocoa": 500 + } + } + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_no_optdata.json b/test/optengine-tests/test_optengine_no_optdata.json new file mode 100644 index 0000000..f6645c8 --- /dev/null +++ b/test/optengine-tests/test_optengine_no_optdata.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];" + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_solverid.json b/test/optengine-tests/test_optengine_solverid.json new file mode 100644 index 0000000..bfd446c --- /dev/null +++ b/test/optengine-tests/test_optengine_solverid.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "py", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_optengine_valid.json b/test/optengine-tests/test_optengine_valid.json new file mode 100644 index 0000000..8de2b71 --- /dev/null +++ b/test/optengine-tests/test_optengine_valid.json @@ -0,0 +1,20 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "mzn", + "solverArgs": { + "solver": "geocode" + }, + "modelContent": "int: nc;\r\nvar 1 .. nc: wa; var 1 .. nc: nt; var 1 .. nc: sa; var 1 .. nc: q;\r\nvar 1 .. nc: nsw; var 1 .. nc: v; var 1 .. nc: t;\r\nconstraint wa != nt;\r\nconstraint wa != sa;\r\nconstraint nt != sa;\r\nconstraint nt != q;\r\nconstraint sa != q;\r\nconstraint sa != nsw;\r\nconstraint sa != v;\r\nconstraint q != nsw;\r\nconstraint nsw != v;\r\nsolve satisfy;\r\noutput [\r\n \"wa=\\(wa)\\t nt=\\(nt)\\t sa=\\(sa)\\n\",\r\n \"q=\\(q)\\t nsw=\\(nsw)\\t v=\\(v)\\n\",\r\n \"t=\", show(t), \"\\n\"\r\n];", + "optData": { + "json": { + "nc": 3 + } + } + } +} \ No newline at end of file diff --git a/test/optengine-tests/test_py_optengine_valid.json b/test/optengine-tests/test_py_optengine_valid.json new file mode 100644 index 0000000..bfd446c --- /dev/null +++ b/test/optengine-tests/test_py_optengine_valid.json @@ -0,0 +1,15 @@ +{ + "requestInfo": { + "transactionId": "xxx-xxx-xxxx", + "requestID": "yyy-yyy-yyyy", + "sourceId": "cmopt", + "timeout": 600 + }, + "optimInfo": { + "solver": "py", + "modelContent": "import sys\r\n\r\nif __name__ == \"__main__\":\r\n print(sys.argv[1],sys.argv[2])\r\n\r\n with open(sys.argv[2], \"wt\") as f:\r\n f.write('{\"hello\":\"world\",\"another\":\"string\"}')\r\n\r\n", + "optData": { + "text": "flour = 8000; \r\nbanana = 11; \r\nsugar = 3000; \r\nbutter = 1500; \r\ncocoa = 800; " + } + } +} \ No newline at end of file diff --git a/test/test_model_api.py b/test/test_model_api.py new file mode 100644 index 0000000..2a1cecf --- /dev/null +++ b/test/test_model_api.py @@ -0,0 +1,71 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import os + +import pytest +from mock import patch +from schematics.exceptions import DataError + +from runtime.model_api import create_model_data, get_model_data, delete_model_data, retrieve_all_models +from runtime.models.api.model_request import OptimModelRequestAPI +from runtime.optim_engine import validate_request + +BASE_DIR = os.path.dirname(__file__) + +ret_val = {'modelId': 'test', 'description': 'desc', 'solver': 'mzn'} + + +class TestModelApi(): + + def test_valid_mapi_request(self): + req_json = json.loads(open("./test/optengine-tests/test_modelapi_valid.json").read()) + + assert OptimModelRequestAPI(req_json).validate() is None + + def test_invalid_mapi_request(self): + req_json = json.loads(open("./test/optengine-tests/test_modelapi_invalid.json").read()) + with pytest.raises(DataError): + validate_request(req_json) + + @patch('runtime.model_api.build_model_dict') + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_create_model(self, config, conn, model_data): + model_data.return_value = ret_val + req_json = json.loads(open("./test/optengine-tests/test_modelapi_valid.json").read()) + + create_model_data(req_json) + + @patch('runtime.model_api.build_model_dict') + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_retrieve_model(self, config, conn, model_data): + model_data.return_value = ret_val + get_model_data('test') + + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_delete_model(self, config, conn): + delete_model_data('test') + + @patch('mysql.connector.connect') + @patch('runtime.model_api.osdf_config') + def test_retrieve_all_model(self, config, conn): + retrieve_all_models() diff --git a/test/test_optim_engine.py b/test/test_optim_engine.py new file mode 100644 index 0000000..e1756f8 --- /dev/null +++ b/test/test_optim_engine.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------------------- +# Copyright (c) 2020 AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +import json +import os + +import pytest +from mock import patch +from schematics.exceptions import DataError + +from osdf.operation.exceptions import BusinessException +from runtime.optim_engine import validate_request, process_request + +BASE_DIR = os.path.dirname(__file__) + + +class TestOptimEngine(): + + def test_valid_optim_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_valid.json").read()) + + assert validate_request(req_json) == True + + def test_invalid_optim_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid.json").read()) + with pytest.raises(DataError): + validate_request(req_json) + + def test_invalid_optim_request_without_modelid(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid2.json").read()) + with pytest.raises(BusinessException): + validate_request(req_json) + + def test_invalid_optim_request_no_optdata(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_no_optdata.json").read()) + with pytest.raises(BusinessException): + validate_request(req_json) + + def test_process_request(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_valid.json").read()) + + res = process_request(req_json) + assert res.status_code == 400 + + def test_py_process_request(self): + req_json = json.loads(open("./test/optengine-tests/test_py_optengine_valid.json").read()) + + res = process_request(req_json) + assert res.status_code == 200 + + def test_invalid_solver(self): + req_json = json.loads(open("./test/optengine-tests/test_optengine_invalid_solver.json").read()) + + with pytest.raises(BusinessException): + process_request(req_json) + + @patch('runtime.optim_engine.get_model_data') + def test_process_solverid_request(self, mocker): + req_json = json.loads(open("./test/optengine-tests/test_optengine_modelId.json").read()) + + data = 200, ('junk', '', '', 'py') + mocker.return_value = data + process_request(req_json) diff --git a/tox.ini b/tox.ini index 2c30e73..7b0fb07 100644 --- a/tox.ini +++ b/tox.ini @@ -17,13 +17,14 @@ commands = # TODO: need to update the above "omit" when we package osdf as pip-installable deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test/test-requirements.txt + -r{toxinidir}/requirements-opteng.txt [run] -source=./apps/,./osdf/,osdfapp.py +source=./apps/,./osdf/,osdfapp.py,./runtime/,solverapp.py [testenv:pylint] whitelist_externals=bash -commands = bash -c "pylint --reports=y osdf apps| tee pylint.out" +commands = bash -c "pylint --reports=y osdf apps runtime| tee pylint.out" [testenv:py3] basepython=python3.6 -- cgit 1.2.3-korg