diff options
Diffstat (limited to 'vnfs')
12 files changed, 446 insertions, 0 deletions
diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/.helmignore b/vnfs/DAaaS/sample-apps/sample-minio-load-model/.helmignore new file mode 100644 index 00000000..33aaef25 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/.helmignore @@ -0,0 +1,24 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +build
\ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/Chart.yaml b/vnfs/DAaaS/sample-apps/sample-minio-load-model/Chart.yaml new file mode 100644 index 00000000..084b7d80 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: model-loader +version: 0.1.0 diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore new file mode 100644 index 00000000..98bc4363 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/.dockerignore @@ -0,0 +1,12 @@ +.git +.ipynb_checkpoints/* +Dockerfile +.DS_Store +.gitignore +README.md +env.* +/devops/* + +# To prevent storing dev/temporary container data +*.csv +/tmp/*
\ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile new file mode 100644 index 00000000..f7670c2c --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/Dockerfile @@ -0,0 +1,75 @@ +# Copyright (c) 2019 Intel Corporation +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +FROM ubuntu:18.04 + +RUN apt update -yqq + +# Install all the essentials +RUN apt-get update --fix-missing && \ + apt-get install -y --no-install-recommends wget curl ca-certificates libglib2.0-0 libxext6 libsm6 libxrender1 \ + git build-essential openssh-server openssh-client && \ + mkdir -p /var/run/sshd && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PATH /opt/conda/bin:$PATH + +# Install miniconda +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \ + /bin/bash ~/miniconda.sh -b -p /opt/conda && \ + rm ~/miniconda.sh && \ + ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \ + echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \ + echo "conda activate base" >> ~/.bashrc + +# Install python libraries using conda in the virtual_environment:ml_env +SHELL ["/bin/bash", "-c"] +RUN conda update -n base -c defaults conda && \ + conda create -n ml_env +RUN conda install -n ml_env -y -c anaconda pip +RUN pwd +RUN conda install -n ml_env -y -c anaconda h5py + +RUN conda install -n ml_env -y -c pytorch pytorch-cpu +RUN conda install -n ml_env -y -c conda-forge nlopt + +RUN echo "conda activate ml_env" >> ~/.bashrc +RUN source ~/.bashrc +RUN /opt/conda/envs/ml_env/bin/pip install --no-cache-dir minio + +# Install tini +RUN apt-get install -y --no-install-recommends curl grep sed dpkg && \ + TINI_VERSION=`curl https://github.com/krallin/tini/releases/latest | grep -o "/v.*\"" | sed 's:^..\(.*\).$:\1:'` && echo ${TINI_VERSION} && \ + curl -L "https://github.com/krallin/tini/releases/download/v${TINI_VERSION}/tini_${TINI_VERSION}.deb" > tini.deb && \ + dpkg -i tini.deb && \ + rm tini.deb && \ + apt clean + +# This is needed to match the original entrypoint.sh file. +RUN cp /usr/bin/tini /sbin + + +RUN echo "export PATH=/opt/conda/envs/ml_env/bin:$PATH" >> ~/.bashrc + +COPY . /app +WORKDIR /app +RUN source ~/.bashrc +RUN conda install -n ml_env -y --file requirements.txt + +ENTRYPOINT [ "/app/entrypoint.sh" ] diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh new file mode 100755 index 00000000..fe52a9f1 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/entrypoint.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# echo commands to the terminal output +set -ex + +# Check whether there is a passwd entry for the container UID +myuid=$(id -u) +mygid=$(id -g) +# turn off -e for getent because it will return error code in anonymous uid case +set +e +uidentry=$(getent passwd $myuid) +set -e + +# If there is no passwd entry for the container UID, attempt to create one +if [ -z "$uidentry" ] ; then + if [ -w /etc/passwd ] ; then + echo "$myuid:x:$myuid:$mygid:anonymous uid:$SPARK_HOME:/bin/false" >> /etc/passwd + else + echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID" + fi +fi + +CMD=( + /bin/bash -c "source activate ml_env && exec python load-model-from-minio.py") + +# Execute the container CMD under tini for better hygiene +exec /sbin/tini -s -- "${CMD[@]}" diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py new file mode 100755 index 00000000..dbdabc3a --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/load-model-from-minio.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# +# ------------------------------------------------------------------------- +# Copyright (c) 2019 Intel Corporation Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ------------------------------------------------------------------------- +# + +'''Sample Python application to load model from Minio''' + +# python imports + +import json +import os +import time +from minio import Minio +from minio.error import ResponseError + + +client = Minio(os.environ['S3_ENDPOINT'], + access_key=os.environ['AWS_ACCESS_KEY_ID'], + secret_key=os.environ['AWS_SECRET_ACCESS_KEY'], + secure=False) + + +def load_model(bucket_name, object_name, filepath): + try: + client.fget_object(bucket_name, object_name, filepath) + except ResponseError as err: + print(err) + +if __name__ == "__main__": + bucket_name = os.environ['MODEL_BUCKET'] + objects = json.loads(os.environ['MODEL_OBJECTS']) + filepath = '/models/' + for object_name in objects: + print(f"Load model from s3://{bucket_name}/{object_name} to {filepath+object_name}") + load_model(bucket_name, object_name, filepath+object_name) + time.sleep(300)
\ No newline at end of file diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt new file mode 100644 index 00000000..475fbd5a --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/build/requirements.txt @@ -0,0 +1,3 @@ +numpy +scipy +pandas diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/_helpers.tpl b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/_helpers.tpl new file mode 100755 index 00000000..5aa78da6 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "model-loader.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "model-loader.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "model-loader.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account +*/}} +{{- define "model-loader.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "model-loader.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/deployment.yaml b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/deployment.yaml new file mode 100644 index 00000000..53e07389 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/deployment.yaml @@ -0,0 +1,79 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ include "model-loader.fullname" . }} + labels: + app: {{ include "model-loader.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service | quote }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "model-loader.name" . }} + release: {{ .Release.Name }} + modelName: {{ .Values.modelName }} + template: + metadata: + labels: + app: {{ include "model-loader.name" . }} + release: {{ .Release.Name }} + modelName: {{ .Values.modelName }} + spec: + serviceAccountName: {{ template "model-loader.serviceAccountName" . }} + containers: + - name: model-loader + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "model-loader.fullname" . }}{{ end }} + key: accesskey + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "model-loader.fullname" . }}{{ end }} + key: secretkey + {{- range $key, $val := .Values.minio.environment }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end}} + resources: + {{ toYaml .Values.resources | indent 12 }} + volumeMounts: + - mountPath: /models + name: models + volumes: + - name: models + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/secrets.yaml b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/secrets.yaml new file mode 100644 index 00000000..60717a65 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/secrets.yaml @@ -0,0 +1,31 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +{{- if not .Values.minio.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "model-loader.fullname" . }} + labels: + app: {{ include "model-loader.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service | quote }} +type: Opaque +data: + accesskey: {{ .Values.minio.accessKey | b64enc }} + secretkey: {{ .Values.minio.secretKey | b64enc }} +{{- end }} diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/serviceaccount.yaml b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/serviceaccount.yaml new file mode 100644 index 00000000..1573ae94 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/templates/serviceaccount.yaml @@ -0,0 +1,25 @@ +{{/* +# Copyright 2019 Intel Corporation, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "model-loader.fullname" . }} + labels: + app: {{ include "model-loader.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service | quote }} diff --git a/vnfs/DAaaS/sample-apps/sample-minio-load-model/values.yaml b/vnfs/DAaaS/sample-apps/sample-minio-load-model/values.yaml new file mode 100644 index 00000000..2b94c1c8 --- /dev/null +++ b/vnfs/DAaaS/sample-apps/sample-minio-load-model/values.yaml @@ -0,0 +1,55 @@ +# Default values for model-loader. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: dcr.cluster.local:32644/sample-minio-load + tag: v3 + pullPolicy: Always + +nameOverride: "" +fullnameOverride: "" + +## Model repository information (Minio) +## Model repository information (Minio) +minio: + existingSecret: "" + accessKey: "onapdaas" + secretKey: "onapsecretdaas" + environment: + AWS_REGION: "us-west-1" + S3_REGION: "us-west-1" + S3_ENDPOINT: "minio.edge1.svc.cluster.local:9000" + AWS_ENDPOINT_URL: "http://minio.edge1.svc.cluster.local:9000" + S3_USE_HTTPS: 0 + S3_VERIFY_SSL: 0 + AWS_LOG_LEVEL: 3 + TF_CPP_MIN_LOG_LEVEL: 3 + MODEL_BUCKET: "numpy" + # List of object files to be loaded given in a , separated list format + MODEL_OBJECTS: '["model1.data", "model2.data"]' + +## Service account for model-loader to use. +serviceAccount: + create: true + name: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} |