Add prometheus server to k8s charts

- add pap-slas.test to list of robot tests for k8s
- fix an export var on docker compose

Issue-ID: POLICY-4545
Change-Id: Id1dd55693ab6d016fd0dd38230972ffe96b28b14
Signed-off-by: adheli.tavares <adheli.tavares@est.tech>

10 files changed, 335 insertions, 95 deletions

diff --git a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml b/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml
deleted file mode 100644
index 3e7bad66..00000000
--- a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-{{/*
-# Copyright © 2022 Nordix Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/}}
-
-{{- if default false (and .Values.global.metrics.enabled .Values.global.metrics.custom_resources) }}
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: { .Chart.Name }}
-  namespace: {{ include "common.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ .Chart.Name }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: Helm
-
-    {{- toYaml .Values.metrics.prometheusRules.selector | nindent 4 }}
-spec:
-  groups:
-  - name: {{ .Chart.Name }}
-    rules:
-      {{- toYaml .Values.metrics.prometheusRules.rules | nindent 6 }}
-{{- end }}
-
diff --git a/helm/policy/components/mariadb-galera/templates/statefulset.yaml b/helm/policy/components/mariadb-galera/templates/statefulset.yaml
index 3644a9bd..c347bef0 100644
--- a/helm/policy/components/mariadb-galera/templates/statefulset.yaml
+++ b/helm/policy/components/mariadb-galera/templates/statefulset.yaml
@@ -223,52 +223,6 @@ spec:
             - name: mariadb-galera-actual-config
               mountPath: /opt/bitnami/mariadb/conf
             {{- end }}
-        {{- if default false .Values.global.metrics.enabled }}
-        - name: {{ .Chart.Name }}-metrics
-          image: {{ .Values.metrics.image }}
-          imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.metrics.pullPolicy | quote}}
-          env:
-            - name: MARIADB_METRICS_EXTRA_FLAGS
-              value: {{ default "" (join " " .Values.metrics.extraFlags) | quote }}
-            - name: MARIADB_ROOT_USER
-              value: {{ .Values.rootUser.user | quote }}
-            - name: MARIADB_ROOT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mariadb-galera-db-root-password
-                  key: password
-          command:
-            - sh
-            - -c
-            - |
-              DATA_SOURCE_NAME="$MARIADB_ROOT_USER:$MARIADB_ROOT_PASSWORD@(localhost:3306)/" /bin/mysqld_exporter $MARIADB_METRICS_EXTRA_FLAGS
-          ports:
-            - name: tcp-metrics
-              containerPort: 9104
-          livenessProbe:
-            httpGet:
-              path: /metrics
-              port: tcp-metrics
-            initialDelaySeconds: {{ .Values.metrics.livenessProbe.initialDelaySeconds }}
-            periodSeconds: {{ .Values.metrics.livenessProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.metrics.livenessProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.metrics.livenessProbe.successThreshold }}
-            failureThreshold: {{ .Values.metrics.livenessProbe.failureThreshold }}
-          readinessProbe:
-            httpGet:
-              path: /metrics
-              port: tcp-metrics
-            initialDelaySeconds: {{ .Values.metrics.readinessProbe.initialDelaySeconds }}
-            periodSeconds: {{ .Values.metrics.readinessProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.metrics.readinessProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.metrics.readinessProbe.successThreshold }}
-            failureThreshold: {{ .Values.metrics.readinessProbe.failureThreshold }}
-          securityContext:
-            runAsUser: 10001
-            runAsGroup: 10001
-            fsGroup: 10001
-          resources: {{- toYaml .Values.metrics.resources | nindent 12 }}
-        {{- end }}
       imagePullSecrets:
         - name: default-docker-registry-key
       {{- if .Values.schedulerName }}
diff --git a/helm/prometheus/Chart.yaml b/helm/prometheus/Chart.yaml
new file mode 100644
index 00000000..75e5a2b7
--- /dev/null
+++ b/helm/prometheus/Chart.yaml
@@ -0,0 +1,27 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v2
+name: prometheus
+description: Prometheus server to collect all policy components metrics
+keywords:
+- prometheus
+- metrics
+- monitoring
+type: application
+version: 0.1.0
diff --git a/helm/prometheus/resources/prometheus.yml b/helm/prometheus/resources/prometheus.yml
new file mode 100644
index 00000000..70041c97
--- /dev/null
+++ b/helm/prometheus/resources/prometheus.yml
@@ -0,0 +1,96 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+# global config
+global:
+  scrape_interval: 60s
+  evaluation_interval: 10s
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# scrape config
+scrape_configs:
+  - job_name: "api-metrics"
+    metrics_path: /policy/api/v1/metrics
+    static_configs:
+      - targets: ["policy-api:6969"]
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "pap-metrics"
+    metrics_path: /policy/pap/v1/metrics
+    static_configs:
+      - targets: ["policy-pap:6969"]
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "apex-pdp-metrics"
+    static_configs:
+      - targets:
+          - "policy-apex-pdp:6969"
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  # - job_name: "drools-apps-metrics"
+  #   static_configs:
+  #     - targets:
+  #         - "drools-apps:9696"
+  #   basic_auth:
+  #     username: "demo@people.osaaf.org"
+  #     password: "demo123456!"
+
+  - job_name: "drools-pdp-metrics"
+    static_configs:
+      - targets:
+          - "policy-drools-pdp:9696"
+    basic_auth:
+      username: "demo@people.osaaf.org"
+      password: "demo123456!"
+
+  - job_name: "distribution-metrics"
+    static_configs:
+      - targets:
+          - "policy-distribution:6969"
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "xacml-pdp-metrics"
+    static_configs:
+      - targets:
+          - "policy-xacml-pdp:6969"
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "acm-metrics"
+    metrics_path: "/onap/policy/clamp/acm/prometheus"
+    static_configs:
+      - targets:
+          - "policy-clamp-runtime-acm:6969"
+    basic_auth:
+      username: "runtimeUser"
+      password: "zb!XztG34"
diff --git a/helm/prometheus/templates/cluster-role.yaml b/helm/prometheus/templates/cluster-role.yaml
new file mode 100644
index 00000000..d8ce874c
--- /dev/null
+++ b/helm/prometheus/templates/cluster-role.yaml
@@ -0,0 +1,52 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Chart.Name }}
+  namespace: default
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs: ["get", "list", "watch"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Chart.Name }}-read
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Chart.Name }}-read
+subjects:
+- kind: ServiceAccount
+  name: {{ .Chart.Name }}-read
+  namespace: default
diff --git a/helm/prometheus/templates/configmap.yaml b/helm/prometheus/templates/configmap.yaml
new file mode 100644
index 00000000..05c2964b
--- /dev/null
+++ b/helm/prometheus/templates/configmap.yaml
@@ -0,0 +1,35 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Chart.Name }}-configmap
+  namespace: default
+  labels:
+    app: {{ .Chart.Name }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    heritage: Helm
+{{- with .Files.Glob "resources/*store" }}
+binaryData:
+{{- range $path, $bytes := . }}
+  {{ base $path }}: {{ $.Files.Get $path | b64enc | quote }}
+{{- end }}
+{{- end }}
+data:
+{{ tpl (.Files.Glob "resources/*.{yaml,yml}").AsConfig . | indent 2 }}
\ No newline at end of file
diff --git a/helm/prometheus/templates/deployment.yaml b/helm/prometheus/templates/deployment.yaml
new file mode 100644
index 00000000..818af85f
--- /dev/null
+++ b/helm/prometheus/templates/deployment.yaml
@@ -0,0 +1,57 @@
+
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ .Chart.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ .Chart.Name }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+        app.kubernetes.io/instance: RELEASE
+        app.kubernetes.io/managed-by: Helm
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9090"
+    spec:
+      containers:
+        - image: prom/prometheus:v2.32.1
+          name: prometheus
+          args:
+          - '--config.file=/etc/prometheus/prometheus.yml'
+          ports:
+            - containerPort: 9090
+          resources: {}
+          volumeMounts:
+            - mountPath: /etc/prometheus
+              name: prometheusconfig
+      hostname: prometheus
+      restartPolicy: Always
+      volumes:
+        - name: prometheusconfig
+          configMap:
+              name: {{ .Chart.Name }}-configmap
+              defaultMode: 0755
diff --git a/helm/prometheus/templates/role-binding.yaml b/helm/prometheus/templates/role-binding.yaml
new file mode 100644
index 00000000..355e4eda
--- /dev/null
+++ b/helm/prometheus/templates/role-binding.yaml
@@ -0,0 +1,30 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Chart.Name }}-read
+  namespace: default
+subjects:
+- kind: ServiceAccount
+  name: {{ .Chart.Name }}-read
+roleRef:
+  kind: Role
+  name: read
+  apiGroup: rbac.authorization.k8s.io
diff --git a/helm/prometheus/templates/service-account.yaml b/helm/prometheus/templates/service-account.yaml
new file mode 100644
index 00000000..3f34b5b9
--- /dev/null
+++ b/helm/prometheus/templates/service-account.yaml
@@ -0,0 +1,23 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Chart.Name }}-read
+  namespace: default
diff --git a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml b/helm/prometheus/templates/service.yaml
index 00917452..2ce3b028 100644
--- a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml
+++ b/helm/prometheus/templates/service.yaml
@@ -1,37 +1,38 @@
-{{/*
-# Copyright © 2022 Nordix Foundation
 #
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+#      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-*/}}
+# ============LICENSE_END=====================================================
+#
 
-{{- if default false .Values.global.metrics.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Chart.Name }}-metrics
+  name: {{ .Chart.Name }}
   namespace: default
   labels:
     app.kubernetes.io/name: {{ .Chart.Name }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: Helm
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '9090'
 spec:
-  type: {{ .Values.metrics.service.type }}
+  type: NodePort
   ports:
-    - name: tcp-metrics
-      port: {{ .Values.metrics.service.port }}
-      targetPort: tcp-metrics
+    - port: 9090
+      targetPort: 9090
+      nodePort: 30909
   selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ .Chart.Name }}
-
-{{- end }}
+    app.kubernetes.io/name: {{ .Chart.Name }}