From 3d5a3e06530c1250d48f7d838c619f3bfbcd019d Mon Sep 17 00:00:00 2001 From: Dileep Ranganathan Date: Thu, 30 May 2019 12:38:37 -0700 Subject: Refactor Distributed Analytics project structure Modified the project structure to improve maintainability and to add future CI and integration test support. Change-Id: Id30bfb1f83f23785a6b5f99e81f42f752d59c0f8 Issue-ID: ONAPARC-280 Signed-off-by: Dileep Ranganathan --- vnfs/DAaaS/inference-core/.helmignore | 22 ---- vnfs/DAaaS/inference-core/Chart.yaml | 5 - .../inference-core/charts/tf-serving/.helmignore | 22 ---- .../inference-core/charts/tf-serving/Chart.yaml | 5 - .../charts/tf-serving/templates/NOTES.txt | 20 --- .../charts/tf-serving/templates/_helpers.tpl | 41 ------ .../charts/tf-serving/templates/deployment.yaml | 138 --------------------- .../charts/tf-serving/templates/ingress.yaml | 55 -------- .../charts/tf-serving/templates/secrets.yaml | 31 ----- .../charts/tf-serving/templates/service.yaml | 39 ------ .../tf-serving/templates/serviceaccount.yaml | 25 ---- .../inference-core/charts/tf-serving/values.yaml | 71 ----------- vnfs/DAaaS/inference-core/values.yaml | 30 ----- 13 files changed, 504 deletions(-) delete mode 100644 vnfs/DAaaS/inference-core/.helmignore delete mode 100644 vnfs/DAaaS/inference-core/Chart.yaml delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/.helmignore delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/Chart.yaml delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/templates/NOTES.txt delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/templates/_helpers.tpl delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/templates/deployment.yaml delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/templates/ingress.yaml delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/templates/secrets.yaml delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/templates/service.yaml delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/templates/serviceaccount.yaml delete mode 100644 vnfs/DAaaS/inference-core/charts/tf-serving/values.yaml delete mode 100644 vnfs/DAaaS/inference-core/values.yaml (limited to 'vnfs/DAaaS/inference-core') diff --git a/vnfs/DAaaS/inference-core/.helmignore b/vnfs/DAaaS/inference-core/.helmignore deleted file mode 100644 index 50af0317..00000000 --- a/vnfs/DAaaS/inference-core/.helmignore +++ /dev/null @@ -1,22 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/vnfs/DAaaS/inference-core/Chart.yaml b/vnfs/DAaaS/inference-core/Chart.yaml deleted file mode 100644 index 836b1a84..00000000 --- a/vnfs/DAaaS/inference-core/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: A Helm chart for inference framework components -name: inference-core -version: 0.1.0 diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/.helmignore b/vnfs/DAaaS/inference-core/charts/tf-serving/.helmignore deleted file mode 100644 index 50af0317..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/.helmignore +++ /dev/null @@ -1,22 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/Chart.yaml b/vnfs/DAaaS/inference-core/charts/tf-serving/Chart.yaml deleted file mode 100644 index 2db1a483..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "1.0" -description: Helm chart for Tensorflow serving model server -name: tf-serving -version: 0.1.0 diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/NOTES.txt b/vnfs/DAaaS/inference-core/charts/tf-serving/templates/NOTES.txt deleted file mode 100644 index 2dcf639e..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/NOTES.txt +++ /dev/null @@ -1,20 +0,0 @@ -1. Get the tensorflow serving URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range $.Values.ingress.paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host }}{{ . }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "tf-serving.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get svc -w {{ include "tf-serving.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "tf-serving.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export SVC_NAME=$(kubectl get svc --namespace {{ .Release.Namespace }} -l "app={{ include "tf-serving.name" . }},modelName={{ .Values.modelName }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - kubectl port-forward svc/$SVC_NAME 8500 -{{- end }} diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/_helpers.tpl b/vnfs/DAaaS/inference-core/charts/tf-serving/templates/_helpers.tpl deleted file mode 100644 index ce3ce917..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/_helpers.tpl +++ /dev/null @@ -1,41 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "tf-serving.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "tf-serving.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.modelName | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "tf-serving.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* Create the name of tf-serving service account to use */}} -{{- define "tf-serving.serviceAccountName" -}} -{{- if and .Values.global.rbac .Values.serviceAccount.create -}} - {{ default (include "tf-serving.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} \ No newline at end of file diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/deployment.yaml b/vnfs/DAaaS/inference-core/charts/tf-serving/templates/deployment.yaml deleted file mode 100644 index 0a909e9f..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/deployment.yaml +++ /dev/null @@ -1,138 +0,0 @@ -{{/* -# Copyright 2019 Intel Corporation, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: {{ include "tf-serving.fullname" . }} - labels: - app: {{ include "tf-serving.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service | quote }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "tf-serving.name" . }} - release: {{ .Release.Name }} - modelName: {{ .Values.modelName }} - template: - metadata: - labels: - app: {{ include "tf-serving.name" . }} - release: {{ .Release.Name }} - modelName: {{ .Values.modelName }} - spec: - serviceAccountName: {{ template "tf-serving.serviceAccountName" . }} - containers: - - name: tf-serving - image: "{{ .Values.image.tensorflowServing.repository }}:{{ .Values.image.tensorflowServing.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - "/usr/bin/tensorflow_model_server" - args: - - "--port={{ .Values.service.tensorflowServingPort }}" - - "--model_name={{ .Values.modelName }}" - - "--model_base_path={{ .Values.modelBasePath }}" - ports: - - name: tf-serving - containerPort: {{ .Values.service.tensorflowServingPort }} - protocol: TCP - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "tf-serving.fullname" . }}{{ end }} - key: accesskey - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "tf-serving.fullname" . }}{{ end }} - key: secretkey - {{- range $key, $val := .Values.minio.environment }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end}} - readinessProbe: - tcpSocket: - port: tf-serving - initialDelaySeconds: 15 - timeoutSeconds: 1 - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - mountPath: /models - name: models - - name: tensorboard - image: "{{ .Values.image.tensorboard.repository }}:{{ .Values.image.tensorboard.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - tensorboard - args: - - --logdir - - {{ .Values.modelBasePath }} - - --host - - 0.0.0.0 - ports: - - name: tensorboard - containerPort: {{ .Values.service.tensorboardPort }} - protocol: TCP - env: - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "tf-serving.fullname" . }}{{ end }} - key: accesskey - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ if .Values.minio.existingSecret }}{{ .Values.minio.existingSecret }}{{ else }}{{ template "tf-serving.fullname" . }}{{ end }} - key: secretkey - {{- range $key, $val := .Values.minio.environment }} - - name: {{ $key }} - value: {{ $val | quote }} - {{- end}} - livenessProbe: - httpGet: - path: / - port: {{ .Values.service.tensorboardPort }} - readinessProbe: - httpGet: - path: / - port: {{ .Values.service.tensorboardPort }} - volumeMounts: - - mountPath: /output/training_logs - name: training-logs-volume - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumes: - - name: models - emptyDir: {} - - name: training-logs-volume - emptyDir: {} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/ingress.yaml b/vnfs/DAaaS/inference-core/charts/tf-serving/templates/ingress.yaml deleted file mode 100644 index b02fc8bb..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/ingress.yaml +++ /dev/null @@ -1,55 +0,0 @@ -{{/* -# Copyright 2019 Intel Corporation, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "tf-serving.fullname" . -}} -{{- $ingressPaths := .Values.ingress.paths -}} -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - app: {{ include "tf-serving.name" . }} - release: {{ .Release.Name }} - modelName: {{ .Values.modelName }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: -{{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} -{{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ . | quote }} - http: - paths: - {{- range $ingressPaths }} - - path: {{ . }} - backend: - serviceName: {{ $fullName }} - servicePort: http - {{- end }} - {{- end }} -{{- end }} diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/secrets.yaml b/vnfs/DAaaS/inference-core/charts/tf-serving/templates/secrets.yaml deleted file mode 100644 index f4b8fe89..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/secrets.yaml +++ /dev/null @@ -1,31 +0,0 @@ -{{/* -# Copyright 2019 Intel Corporation, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -{{- if not .Values.minio.existingSecret }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "tf-serving.fullname" . }} - labels: - app: {{ include "tf-serving.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service | quote }} -type: Opaque -data: - accesskey: {{ .Values.minio.accessKey | b64enc }} - secretkey: {{ .Values.minio.secretKey | b64enc }} -{{- end }} diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/service.yaml b/vnfs/DAaaS/inference-core/charts/tf-serving/templates/service.yaml deleted file mode 100644 index 3ddcca66..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/service.yaml +++ /dev/null @@ -1,39 +0,0 @@ -{{/* -# Copyright 2019 Intel Corporation, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: Service -metadata: - name: {{ include "tf-serving.fullname" . }} - labels: - app: {{ include "tf-serving.name" . }} - release: {{ .Release.Name }} - modelName: {{ .Values.modelName }} -spec: - type: {{ .Values.service.type }} - ports: - - port: {{ .Values.service.tensorflowServingPort }} - targetPort: tf-serving - protocol: TCP - name: tf-serving - - port: {{ .Values.service.tensorboardPort }} - targetPort: tensorboard - protocol: TCP - name: tensorboard - selector: - app: {{ include "tf-serving.name" . }} - release: {{ .Release.Name }} - modelName: {{ .Values.modelName }} diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/serviceaccount.yaml b/vnfs/DAaaS/inference-core/charts/tf-serving/templates/serviceaccount.yaml deleted file mode 100644 index af4987d8..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/templates/serviceaccount.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{/* -# Copyright 2019 Intel Corporation, Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "tf-serving.fullname" . }} - labels: - app: {{ include "tf-serving.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service | quote }} diff --git a/vnfs/DAaaS/inference-core/charts/tf-serving/values.yaml b/vnfs/DAaaS/inference-core/charts/tf-serving/values.yaml deleted file mode 100644 index ebf88683..00000000 --- a/vnfs/DAaaS/inference-core/charts/tf-serving/values.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# Default values for tf-serving. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - tensorflowServing: - repository: tensorflow/serving - tag: latest - tensorboard: - repository: tensorflow/tensorflow - tag: latest - pullPolicy: IfNotPresent - -nameOverride: "" -fullnameOverride: "" - -## Model information for tf-serving -modelName: "mnist" -modelBasePath: "s3://models/mnist" - -## Model repository information (Minio) -minio: - existingSecret: "" - accessKey: "onapdaas" - secretKey: "onapsecretdaas" - environment: - AWS_REGION: "" - S3_REGION: "" - S3_ENDPOINT: "minio.minio.svc.cluster.local:9000" - AWS_ENDPOINT_URL: "http://minio.minio.svc.cluster.local:9000" - S3_USE_HTTPS: 0 - S3_VERIFY_SSL: 0 - - -## Service account for tf-serving to use. -serviceAccount: - create: true - name: "" - -service: - type: ClusterIP - tensorflowServingPort: 8500 - tensorboardPort: 6006 - -ingress: - enabled: false - annotations: {} - paths: [] - hosts: - - chart-example.local - tls: [] - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/vnfs/DAaaS/inference-core/values.yaml b/vnfs/DAaaS/inference-core/values.yaml deleted file mode 100644 index 10714b88..00000000 --- a/vnfs/DAaaS/inference-core/values.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright © 2019 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################################# -# Global configuration defaults. -################################################################# -global: - nodePortPrefix: 310 - rbac: true - repository: nexus3.onap.org:10001 - readinessRepository: oomk8s - readinessImage: readiness-check:2.0.0 - loggingRepository: docker.elastic.co - loggingImage: beats/filebeat:5.5.0 - -################################################################# -# k8s Operator Day-0 configuration defaults. -################################################################# - -- cgit 1.2.3-korg