Sample application to load model from Minio

Sample application with helm charts to load from model repo.

Change-Id: If8d09e64044bc2587c444efd3ac95a451ec361dd
Issue-ID: ONAPARC-460
Signed-off-by: Dileep Ranganathan <dileep.ranganathan@intel.com>

5 files changed, 184 insertions, 0 deletions

diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore
new file mode 100644
index 00000000..98bc4363
--- /dev/null
+++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore
@@ -0,0 +1,12 @@
+.git
+.ipynb_checkpoints/*
+Dockerfile
+.DS_Store
+.gitignore
+README.md
+env.*
+/devops/*
+
+# To prevent storing dev/temporary container data
+*.csv
+/tmp/*
\ No newline at end of file
diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile
new file mode 100644
index 00000000..f7670c2c
--- /dev/null
+++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile
@@ -0,0 +1,75 @@
+# Copyright (c) 2019 Intel Corporation
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+FROM ubuntu:18.04
+
+RUN apt update -yqq 
+
+# Install all the essentials
+RUN apt-get update --fix-missing && \
+    apt-get install -y --no-install-recommends wget curl ca-certificates libglib2.0-0 libxext6 libsm6 libxrender1 \
+                       git build-essential openssh-server openssh-client && \
+    mkdir -p /var/run/sshd && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV PATH /opt/conda/bin:$PATH
+
+# Install miniconda
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
+    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
+    rm ~/miniconda.sh && \
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
+    echo "conda activate base" >> ~/.bashrc
+
+# Install python libraries using conda in the virtual_environment:ml_env
+SHELL ["/bin/bash", "-c"]
+RUN conda update -n base -c defaults conda && \
+    conda create -n ml_env
+RUN conda install -n ml_env -y -c anaconda pip 
+RUN pwd
+RUN conda install -n ml_env -y -c anaconda h5py
+
+RUN conda install -n ml_env -y -c pytorch pytorch-cpu
+RUN conda install -n ml_env -y -c conda-forge nlopt
+
+RUN echo "conda activate ml_env" >> ~/.bashrc
+RUN source ~/.bashrc
+RUN /opt/conda/envs/ml_env/bin/pip install --no-cache-dir minio
+
+# Install tini
+RUN apt-get install -y --no-install-recommends curl grep sed dpkg && \
+    TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && echo ${TINI_VERSION} && \
+    curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \
+    dpkg -i tini.deb && \
+    rm tini.deb && \
+    apt clean
+
+# This is needed to match the original entrypoint.sh file.
+RUN cp /usr/bin/tini /sbin
+
+
+RUN echo "export PATH=/opt/conda/envs/ml_env/bin:$PATH" >> ~/.bashrc
+
+COPY . /app
+WORKDIR /app
+RUN source ~/.bashrc
+RUN conda install -n ml_env -y --file requirements.txt
+
+ENTRYPOINT [ "/app/entrypoint.sh" ]
diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh
new file mode 100755
index 00000000..fe52a9f1
--- /dev/null
+++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# echo commands to the terminal output
+set -ex
+
+# Check whether there is a passwd entry for the container UID
+myuid=$(id -u)
+mygid=$(id -g)
+# turn off -e for getent because it will return error code in anonymous uid case
+set +e
+uidentry=$(getent passwd $myuid)
+set -e
+
+# If there is no passwd entry for the container UID, attempt to create one
+if [ -z "$uidentry" ] ; then
+    if [ -w /etc/passwd ] ; then
+        echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd
+    else
+        echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
+    fi
+fi
+
+CMD=(
+  /bin/bash -c "source activate ml_env && exec python load-model-from-minio.py")
+
+# Execute the container CMD under tini for better hygiene
+exec /sbin/tini -s -- "${CMD[@]}"
diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py
new file mode 100755
index 00000000..dbdabc3a
--- /dev/null
+++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+#
+# -------------------------------------------------------------------------
+#   Copyright (c) 2019 Intel Corporation Intellectual Property
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+# -------------------------------------------------------------------------
+#
+
+'''Sample Python application to load model from Minio'''
+
+# python imports
+
+import json
+import os
+import time
+from minio import Minio
+from minio.error import ResponseError
+
+
+client = Minio(os.environ['S3_ENDPOINT'],
+               access_key=os.environ['AWS_ACCESS_KEY_ID'],
+               secret_key=os.environ['AWS_SECRET_ACCESS_KEY'],
+               secure=False)
+
+
+def load_model(bucket_name, object_name, filepath):    
+    try:
+        client.fget_object(bucket_name, object_name, filepath)
+    except ResponseError as err:
+        print(err)
+
+if __name__ == "__main__":
+    bucket_name = os.environ['MODEL_BUCKET']
+    objects = json.loads(os.environ['MODEL_OBJECTS'])
+    filepath = '/models/'
+    for object_name in objects:
+        print(f"Load model from s3://{bucket_name}/{object_name} to {filepath+object_name}")
+        load_model(bucket_name, object_name, filepath+object_name)
+    time.sleep(300)
\ No newline at end of file
diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt
new file mode 100644
index 00000000..475fbd5a
--- /dev/null
+++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt
@@ -0,0 +1,3 @@
+numpy
+scipy
+pandas