Training package for distributed analytics

Contains helm charts for M3DB initially.
This package also will consist of other training framework related
helm charts like HDFS, Spark with TF etc.

Change-Id: I67885fa4236f92453477b5cb0d3d79ffe14ff669
Issue-ID: ONAPARC-391
Signed-off-by: Dileep Ranganathan <dileep.ranganathan@intel.com>

8 files changed, 367 insertions, 0 deletions

diff --git a/vnfs/DAaaS/training-core/charts/m3db/.helmignore b/vnfs/DAaaS/training-core/charts/m3db/.helmignore
new file mode 100644
index 00000000..50af0317
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/vnfs/DAaaS/training-core/charts/m3db/Chart.yaml b/vnfs/DAaaS/training-core/charts/m3db/Chart.yaml
new file mode 100644
index 00000000..10d9d542
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/Chart.yaml
@@ -0,0 +1,3 @@
+apiVersion: v1
+name: m3db
+version: 0.1.1
diff --git a/vnfs/DAaaS/training-core/charts/m3db/templates/NOTES.txt b/vnfs/DAaaS/training-core/charts/m3db/templates/NOTES.txt
new file mode 100644
index 00000000..ee7ee3d7
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/templates/NOTES.txt
@@ -0,0 +1 @@
+M3DB Cluster  {{ .Values.m3dbCluster.name }} has been created
\ No newline at end of file
diff --git a/vnfs/DAaaS/training-core/charts/m3db/templates/_helpers.tpl b/vnfs/DAaaS/training-core/charts/m3db/templates/_helpers.tpl
new file mode 100644
index 00000000..36544b12
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/templates/_helpers.tpl
@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "m3db.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "m3db.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "m3db.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/vnfs/DAaaS/training-core/charts/m3db/templates/configmap.yaml b/vnfs/DAaaS/training-core/charts/m3db/templates/configmap.yaml
new file mode 100644
index 00000000..d7197ae9
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/templates/configmap.yaml
@@ -0,0 +1,216 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.m3dbCluster.configMapName }}
+data:
+  m3.yml: |+
+    coordinator:
+      listenAddress:
+        type: "config"
+        value: "0.0.0.0:7201"
+      metrics:
+        scope:
+          prefix: "coordinator"
+        prometheus:
+          handlerPath: /metrics
+          listenAddress: 0.0.0.0:7203
+        sanitization: prometheus
+        samplingRate: 1.0
+        extended: none
+      tagOptions:
+        idScheme: quoted
+      local:
+        namespaces:
+        - namespace: "collectd"
+          type: unaggregated
+          retention: 48h
+    db:
+      logging:
+        level: info
+
+      metrics:
+        prometheus:
+          handlerPath: /metrics
+        sanitization: prometheus
+        samplingRate: 1.0
+        extended: detailed
+
+      listenAddress: 0.0.0.0:9000
+      clusterListenAddress: 0.0.0.0:9001
+      httpNodeListenAddress: 0.0.0.0:9002
+      httpClusterListenAddress: 0.0.0.0:9003
+      debugListenAddress: 0.0.0.0:9004
+
+      hostID:
+        resolver: file
+        file:
+          path: /etc/m3db/pod-identity/identity
+          timeout: 5m
+
+      client:
+        writeConsistencyLevel: majority
+        readConsistencyLevel: unstrict_majority
+        writeTimeout: 10s
+        fetchTimeout: 15s
+        connectTimeout: 20s
+        writeRetry:
+            initialBackoff: 500ms
+            backoffFactor: 3
+            maxRetries: 2
+            jitter: true
+        fetchRetry:
+            initialBackoff: 500ms
+            backoffFactor: 2
+            maxRetries: 3
+            jitter: true
+        backgroundHealthCheckFailLimit: 4
+        backgroundHealthCheckFailThrottleFactor: 0.5
+
+      gcPercentage: 100
+
+      writeNewSeriesAsync: true
+      writeNewSeriesLimitPerSecond: 1048576
+      writeNewSeriesBackoffDuration: 2ms
+
+      bootstrap:
+        bootstrappers:
+            - filesystem
+            - commitlog
+            - peers
+            - uninitialized_topology
+        fs:
+            numProcessorsPerCPU: 0.125
+
+      commitlog:
+        flushMaxBytes: 524288
+        flushEvery: 1s
+        queue:
+            calculationType: fixed
+            size: 2097152
+        blockSize: 10m
+
+      fs:
+        filePathPrefix: /var/lib/m3db
+        writeBufferSize: 65536
+        dataReadBufferSize: 65536
+        infoReadBufferSize: 128
+        seekReadBufferSize: 4096
+        throughputLimitMbps: 100.0
+        throughputCheckEvery: 128
+
+      repair:
+        enabled: false
+        interval: 2h
+        offset: 30m
+        jitter: 1h
+        throttle: 2m
+        checkInterval: 1m
+
+      pooling:
+        blockAllocSize: 16
+        type: simple
+        seriesPool:
+            size: 262144
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        blockPool:
+            size: 262144
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        encoderPool:
+            size: 262144
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        closersPool:
+            size: 104857
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        contextPool:
+            size: 262144
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        segmentReaderPool:
+            size: 16384
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        iteratorPool:
+            size: 2048
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        fetchBlockMetadataResultsPool:
+            size: 65536
+            capacity: 32
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        fetchBlocksMetadataResultsPool:
+            size: 32
+            capacity: 4096
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        hostBlockMetadataSlicePool:
+            size: 131072
+            capacity: 3
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        blockMetadataPool:
+            size: 65536
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        blockMetadataSlicePool:
+            size: 65536
+            capacity: 32
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        blocksMetadataPool:
+            size: 65536
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        blocksMetadataSlicePool:
+            size: 32
+            capacity: 4096
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        identifierPool:
+            size: 262144
+            lowWatermark: 0.7
+            highWatermark: 1.0
+        bytesPool:
+            buckets:
+                - capacity: 16
+                  size: 524288
+                  lowWatermark: 0.7
+                  highWatermark: 1.0
+                - capacity: 32
+                  size: 262144
+                  lowWatermark: 0.7
+                  highWatermark: 1.0
+                - capacity: 64
+                  size: 131072
+                  lowWatermark: 0.7
+                  highWatermark: 1.0
+                - capacity: 128
+                  size: 65536
+                  lowWatermark: 0.7
+                  highWatermark: 1.0
+                - capacity: 256
+                  size: 65536
+                  lowWatermark: 0.7
+                  highWatermark: 1.0
+                - capacity: 1440
+                  size: 16384
+                  lowWatermark: 0.7
+                  highWatermark: 1.0
+                - capacity: 4096
+                  size: 8192
+                  lowWatermark: 0.7
+                  highWatermark: 1.0
+      config:
+        service:
+            env: default_env
+            zone: embedded
+            service: m3db
+            cacheDir: /var/lib/m3kv
+            etcdClusters:
+            - zone: embedded
+              endpoints:
+              - http://{{ .Release.Name }}-{{ .Values.etcdCluster.name }}:2379
diff --git a/vnfs/DAaaS/training-core/charts/m3db/templates/etcd-cluster.yaml b/vnfs/DAaaS/training-core/charts/m3db/templates/etcd-cluster.yaml
new file mode 100644
index 00000000..fcf44256
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/templates/etcd-cluster.yaml
@@ -0,0 +1,20 @@
+apiVersion: "etcd.database.coreos.com/v1beta2"
+kind: "EtcdCluster"
+metadata:
+  name: {{ .Release.Name }}-{{ .Values.etcdCluster.name }}
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    app: {{ template "m3db.name" . }}
+    heritage: {{ .Release.Service }}
+    release: {{ .Release.Name }}
+  annotations:
+     etcd.database.coreos.com/scope: clusterwide
+spec:
+  size: {{ .Values.etcdCluster.size }}
+  version: "{{ .Values.etcdCluster.version }}"
+  pod:
+{{ toYaml .Values.etcdCluster.pod | indent 4 }}
+  {{- if .Values.etcdCluster.enableTLS }}
+  TLS:
+{{ toYaml .Values.etcdCluster.tls | indent 4 }}
+  {{- end }}
diff --git a/vnfs/DAaaS/training-core/charts/m3db/templates/m3dbcluster.yaml b/vnfs/DAaaS/training-core/charts/m3db/templates/m3dbcluster.yaml
new file mode 100644
index 00000000..5e804351
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/templates/m3dbcluster.yaml
@@ -0,0 +1,22 @@
+apiVersion: operator.m3db.io/v1alpha1
+kind: M3DBCluster
+metadata:
+  name: {{ .Values.m3dbCluster.name }}
+spec:
+  image: {{ .Values.m3dbCluster.image.repository }}:{{ .Values.m3dbCluster.image.tag }}
+  replicationFactor: {{ .Values.m3dbCluster.replicationFactor }}
+  numberOfShards: {{ .Values.m3dbCluster.numberOfShards }}
+  isolationGroups:
+{{ toYaml .Values.m3dbCluster.isolationGroups | indent 4 }}
+  namespaces:
+{{ toYaml .Values.m3dbCluster.namespaces | indent 4 }}
+  configMapName: {{ .Values.m3dbCluster.configMapName }}
+  resources:
+    requests:
+      memory: 4Gi
+      cpu: '1'
+    limits:
+      memory: 12Gi
+      cpu: '4'
+
+  
diff --git a/vnfs/DAaaS/training-core/charts/m3db/values.yaml b/vnfs/DAaaS/training-core/charts/m3db/values.yaml
new file mode 100644
index 00000000..ab365cfa
--- /dev/null
+++ b/vnfs/DAaaS/training-core/charts/m3db/values.yaml
@@ -0,0 +1,51 @@
+m3dbCluster:
+  name: m3db-cluster
+  image:
+    repository: quay.io/m3db/m3dbnode
+    tag: latest
+  replicationFactor: 3
+  numberOfShards: 256
+  isolationGroups:
+    - name: us-west1-a
+      numInstances: 1
+    - name: us-west1-b
+      numInstances: 1
+    - name: us-west1-c
+      numInstances: 1
+  namespaces:
+    - name: collectd
+      preset: 10s:2d
+  configMapName: m3-configuration
+
+etcdCluster:
+  name: etcd
+  size: 3
+  version: 3.3.3
+  image:
+    repository: quay.io/coreos/etcd
+    tag: v3.3.3
+    pullPolicy: Always
+  enableTLS: false
+  # TLS configs
+  tls:
+    static:
+      member:
+        peerSecret: etcd-peer-tls
+        serverSecret: etcd-server-tls
+      operatorSecret: etcd-client-tls
+  ## etcd cluster pod specific values
+  ## Ref: https://github.com/coreos/etcd-operator/blob/master/doc/user/spec_examples.md#three-members-cluster-with-resource-requirement
+  pod:
+    ## Antiaffinity for etcd pod assignment
+    ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+    antiAffinity: false
+    resources:
+      limits:
+        cpu: 100m
+        memory: 128Mi
+      requests:
+        cpu: 100m
+        memory: 128Mi
+    ## Node labels for etcd pod assignment
+    ## Ref: https://kubernetes.io/docs/user-guide/node-selection/
+    nodeSelector: {}