From 3d5a3e06530c1250d48f7d838c619f3bfbcd019d Mon Sep 17 00:00:00 2001 From: Dileep Ranganathan Date: Thu, 30 May 2019 12:38:37 -0700 Subject: Refactor Distributed Analytics project structure Modified the project structure to improve maintainability and to add future CI and integration test support. Change-Id: Id30bfb1f83f23785a6b5f99e81f42f752d59c0f8 Issue-ID: ONAPARC-280 Signed-off-by: Dileep Ranganathan --- .../collection/charts/prometheus/.helmignore | 22 ++++++ .../deploy/collection/charts/prometheus/Chart.yaml | 5 ++ .../charts/prometheus/templates/NOTES.txt | 15 ++++ .../charts/prometheus/templates/_helpers.tpl | 47 +++++++++++++ .../charts/prometheus/templates/prometheus.yaml | 47 +++++++++++++ .../charts/prometheus/templates/service.yaml | 38 +++++++++++ .../prometheus/templates/servicemonitor.yaml | 30 ++++++++ .../collection/charts/prometheus/values.yaml | 79 ++++++++++++++++++++++ 8 files changed, 283 insertions(+) create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/.helmignore create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/Chart.yaml create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/templates/NOTES.txt create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/templates/_helpers.tpl create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/templates/prometheus.yaml create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/templates/service.yaml create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/templates/servicemonitor.yaml create mode 100644 vnfs/DAaaS/deploy/collection/charts/prometheus/values.yaml (limited to 'vnfs/DAaaS/deploy/collection/charts/prometheus') diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/.helmignore b/vnfs/DAaaS/deploy/collection/charts/prometheus/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/Chart.yaml b/vnfs/DAaaS/deploy/collection/charts/prometheus/Chart.yaml new file mode 100644 index 00000000..6e7ddfbc --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: Prometheus instance with remote storage integrations. +name: prometheus +version: 0.1.0 diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/NOTES.txt b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/NOTES.txt new file mode 100644 index 00000000..f8882883 --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/NOTES.txt @@ -0,0 +1,15 @@ +1. Get the application URL by running these commands: +{{ if contains "NodePort" .Values.prometheus.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "prometheus.fullname" . }}-prometheus) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.prometheus.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ include "prometheus.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "prometheus.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.prometheus.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "prometheus.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:9090 to use your application" + kubectl port-forward $POD_NAME 9090:80 +{{- end }} diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/_helpers.tpl b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/_helpers.tpl new file mode 100644 index 00000000..1ac77dd8 --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/_helpers.tpl @@ -0,0 +1,47 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "prometheus.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "prometheus.labels" }} +chart: {{ template "prometheus.chartref" . }} +release: {{ .Release.Name | quote }} +heritage: {{ .Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/prometheus.yaml b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/prometheus.yaml new file mode 100644 index 00000000..9c3d84c2 --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/prometheus.yaml @@ -0,0 +1,47 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: {{ template "prometheus.fullname" . }}-prometheus + labels: + app: {{ template "prometheus.name" . }}-prometheus + "helm.sh/hook": post-install + "helm.sh/hook-weight": "2" +spec: + serviceMonitorSelector: + matchLabels: + app: {{ template "prometheus.name" . }}-prometheus + release: {{ .Release.Name }} + serviceMonitorNamespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + + # TODO: Templatizing multiple remote read/write. + # Especially Kafka adapater. + remoteRead: + - url: "{{ .Values.m3db.url }}/api/v1/prom/remote/read" + # To test reading even when local Prometheus has the data + readRecent: true + remoteWrite: + - url: "{{ .Values.m3db.url }}/api/v1/prom/remote/write" + # To differentiate between local and remote storage we will add a storage label + writeRelabelConfigs: + - targetLabel: metrics_storage + replacement: m3db_remote + - url: "{{ .Values.kafkaAdapter.url }}/receive" + containers: + - name: {{ template "prometheus.name" . }}-adapter + image: "{{ .Values.kafkaAdapter.image.repository }}:{{ .Values.kafkaAdapter.image.tag }}" + imagePullPolicy: {{ .Values.kafkaAdapter.image.pullPolicy }} + env: + - name: KAFKA_BROKER_LIST + value: {{ .Values.kafkaAdapter.broker }} + - name: KAFKA_TOPIC + value: {{ .Values.kafkaAdapter.topic }} + - name: SERIALIZATION_FORMAT + value: {{ .Values.kafkaAdapter.serializationFormat }} + - name: PORT + value: {{ quote .Values.kafkaAdapter.port }} + - name: LOG_LEVEL + value: {{ .Values.kafkaAdapter.logLevel }} + resources: +{{ toYaml .Values.kafkaAdapter.resources | indent 6 }} diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/service.yaml b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/service.yaml new file mode 100644 index 00000000..0114ed2e --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "prometheus.fullname" . }}-prometheus + labels: + app: {{ template "prometheus.name" . }}-prometheus +{{- if .Values.prometheus.service.annotations }} + annotations: +{{ toYaml .Values.prometheus.service.annotations | indent 4 }} +{{- end }} +spec: +{{- if .Values.prometheus.service.clusterIP }} + clusterIP: {{ .Values.prometheus.service.clusterIP }} +{{- end }} +{{- if .Values.prometheus.service.externalIPs }} + externalIPs: +{{ toYaml .Values.prometheus.service.externalIPs | indent 4 }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.prometheus.service.loadBalancerIP }} +{{- end }} +{{- if .Values.prometheus.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- range $cidr := .Values.prometheus.service.loadBalancerSourceRanges }} + - {{ $cidr }} + {{- end }} +{{- end }} + ports: + - name: web + {{- if eq .Values.prometheus.service.type "NodePort" }} + nodePort: {{ .Values.global.nodePortPrefix }}{{ .Values.prometheus.service.nodePort }} + {{- end }} + port: 9090 + targetPort: web + selector: + app: prometheus + prometheus: {{ template "prometheus.fullname" . }}-prometheus + type: "{{ .Values.prometheus.service.type }}" diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/servicemonitor.yaml b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/servicemonitor.yaml new file mode 100644 index 00000000..ea2b81b6 --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/templates/servicemonitor.yaml @@ -0,0 +1,30 @@ +{{- if .Values.prometheus.additionalServiceMonitors }} +apiVersion: v1 +kind: List +items: +{{- range .Values.prometheus.additionalServiceMonitors }} + - apiVersion: "monitoring.coreos.com/v1" + kind: ServiceMonitor + metadata: + name: {{ .name }} + "helm.sh/hook": post-install + "helm.sh/hook-weight": "1" + labels: + app: {{ template "prometheus.name" $ }}-prometheus +{{ include "prometheus.labels" $ | indent 8 }} + {{- if .additionalLabels }} +{{ toYaml .additionalLabels | indent 8 }} + {{- end }} + spec: + endpoints: +{{ toYaml .endpoints | indent 8 }} + {{- if .jobLabel }} + jobLabel: {{ .jobLabel }} + {{- end }} + namespaceSelector: + matchNames: + - {{ $.Release.Namespace | quote }} + selector: +{{ toYaml .selector | indent 8 }} release: {{ $.Release.Name | quote }} +{{- end }} +{{- end }} diff --git a/vnfs/DAaaS/deploy/collection/charts/prometheus/values.yaml b/vnfs/DAaaS/deploy/collection/charts/prometheus/values.yaml new file mode 100644 index 00000000..4398f52f --- /dev/null +++ b/vnfs/DAaaS/deploy/collection/charts/prometheus/values.yaml @@ -0,0 +1,79 @@ +## Deploy a Prometheus instance +## +prometheus: + additionalServiceMonitors: + - name: service-monitor-collectd + additionalLabels: + collector: collectd + jobLabel: collectd + selector: + matchLabels: + app: collectd + endpoints: + - port: collectd-prometheus + interval: 10s + path: /metrics + - name: service-monitor-node-exporter + additionalLabels: + collector: prometheus-node-exporter + jobLabel: node-exporter + selector: + matchLabels: + app: prometheus-node-exporter + endpoints: + - port: metrics + interval: 30s + - name: service-monitor-cadvisor + additionalLabels: + collector: cadvisor + jobLabel: cadvisor + selector: + matchLabels: + app: cadvisor + endpoints: + - port: cadvisor-prometheus + interval: 10s + path: /metrics + + service: + annotations: {} + labels: {} + clusterIP: "" + + ## To be used with a proxy extraContainer port + targetPort: 9090 + + ## List of IP addresses at which the Prometheus server service is available + ## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips + ## + externalIPs: [] + + ## Port to expose on each node + ## Only used if service.type is 'NodePort' + ## + nodePort: 90 + + ## Loadbalancer IP + ## Only use if service.type is "loadbalancer" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + ## Service type + ## + type: NodePort + + sessionAffinity: "" +m3db: + url: http://m3coordinator-m3db.m3db-operator.svc.cluster.local:7201 + +kafkaAdapter: + image: + repository: telefonica/prometheus-kafka-adapter + tag: 1.2.0 + pullPolicy: IfNotPresent + url: http://localhost:8080 + port: 8080 + broker: my-cluster-kafka-bootstrap.msg.svc.cluster.local:9092 + topic: orders + serializationFormat: json + logLevel: debug + resources: {} -- cgit 1.2.3-korg