From d3aaa31be6005e9fe7446c7fd9a1492aad8a6fd0 Mon Sep 17 00:00:00 2001 From: Dileep Ranganathan Date: Mon, 3 Jun 2019 15:03:41 -0700 Subject: Sample application to load model from Minio Sample application with helm charts to load from model repo. Change-Id: If8d09e64044bc2587c444efd3ac95a451ec361dd Issue-ID: ONAPARC-460 Signed-off-by: Dileep Ranganathan --- .../sample-minio-load-model/build/.dockerignore | 12 ++++ .../sample-minio-load-model/build/Dockerfile | 75 ++++++++++++++++++++++ .../sample-minio-load-model/build/entrypoint.sh | 43 +++++++++++++ .../build/load-model-from-minio.py | 51 +++++++++++++++ .../sample-minio-load-model/build/requirements.txt | 3 + 5 files changed, 184 insertions(+) create mode 100644 vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore create mode 100644 vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile create mode 100755 vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh create mode 100755 vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py create mode 100644 vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt (limited to 'vnfs/DAaaS/sample-apps/sample-minio-load-model/build') diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore new file mode 100644 index 00000000..98bc4363 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore @@ -0,0 +1,12 @@ +.git +.ipynb_checkpoints/* +Dockerfile +.DS_Store +.gitignore +README.md +env.* +/devops/* + +# To prevent storing dev/temporary container data +*.csv +/tmp/* \ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile new file mode 100644 index 00000000..f7670c2c --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile @@ -0,0 +1,75 @@ +# Copyright (c) 2019 Intel Corporation +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +FROM ubuntu:18.04 + +RUN apt update -yqq + +# Install all the essentials +RUN apt-get update --fix-missing && \ + apt-get install -y --no-install-recommends wget curl ca-certificates libglib2.0-0 libxext6 libsm6 libxrender1 \ + git build-essential openssh-server openssh-client && \ + mkdir -p /var/run/sshd && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PATH /opt/conda/bin:$PATH + +# Install miniconda +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc + +# Install python libraries using conda in the virtual_environment:ml_env +SHELL ["/bin/bash", "-c"] +RUN conda update -n base -c defaults conda && \ + conda create -n ml_env +RUN conda install -n ml_env -y -c anaconda pip +RUN pwd +RUN conda install -n ml_env -y -c anaconda h5py + +RUN conda install -n ml_env -y -c pytorch pytorch-cpu +RUN conda install -n ml_env -y -c conda-forge nlopt + +RUN echo "conda activate ml_env" >> ~/.bashrc +RUN source ~/.bashrc +RUN /opt/conda/envs/ml_env/bin/pip install --no-cache-dir minio + +# Install tini +RUN apt-get install -y --no-install-recommends curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && echo ${TINI_VERSION} && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb && \ + apt clean + +# This is needed to match the original entrypoint.sh file. +RUN cp /usr/bin/tini /sbin + + +RUN echo "export PATH=/opt/conda/envs/ml_env/bin:$PATH" >> ~/.bashrc + +COPY . /app +WORKDIR /app +RUN source ~/.bashrc +RUN conda install -n ml_env -y --file requirements.txt + +ENTRYPOINT [ "/app/entrypoint.sh" ] diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh new file mode 100755 index 00000000..fe52a9f1 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# echo commands to the terminal output +set -ex + +# Check whether there is a passwd entry for the container UID +myuid=$(id -u) +mygid=$(id -g) +# turn off -e for getent because it will return error code in anonymous uid case +set +e +uidentry=$(getent passwd $myuid) +set -e + +# If there is no passwd entry for the container UID, attempt to create one +if [ -z "$uidentry" ] ; then + if [ -w /etc/passwd ] ; then + echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd + else + echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" + fi +fi + +CMD=( + /bin/bash -c "source activate ml_env && exec python load-model-from-minio.py") + +# Execute the container CMD under tini for better hygiene +exec /sbin/tini -s -- "${CMD[@]}" diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py new file mode 100755 index 00000000..dbdabc3a --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 Intel Corporation Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +'''Sample Python application to load model from Minio''' + +# python imports + +import json +import os +import time +from minio import Minio +from minio.error import ResponseError + + +client = Minio(os.environ['S3_ENDPOINT'], + access_key=os.environ['AWS_ACCESS_KEY_ID'], + secret_key=os.environ['AWS_SECRET_ACCESS_KEY'], + secure=False) + + +def load_model(bucket_name, object_name, filepath): + try: + client.fget_object(bucket_name, object_name, filepath) + except ResponseError as err: + print(err) + +if __name__ == "__main__": + bucket_name = os.environ['MODEL_BUCKET'] + objects = json.loads(os.environ['MODEL_OBJECTS']) + filepath = '/models/' + for object_name in objects: + print(f"Load model from s3://{bucket_name}/{object_name} to {filepath+object_name}") + load_model(bucket_name, object_name, filepath+object_name) + time.sleep(300) \ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt new file mode 100644 index 00000000..475fbd5a --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt @@ -0,0 +1,3 @@ +numpy +scipy +pandas -- cgit 1.2.3-korg