Add prometheus server to k8s charts

- add pap-slas.test to list of robot tests for k8s
- fix an export var on docker compose

Issue-ID: POLICY-4545
Change-Id: Id1dd55693ab6d016fd0dd38230972ffe96b28b14
Signed-off-by: adheli.tavares <adheli.tavares@est.tech>

12 files changed, 350 insertions, 102 deletions

diff --git a/compose/start-compose.sh b/compose/start-compose.sh
index 57b55316..a77b4834 100755
--- a/compose/start-compose.sh
+++ b/compose/start-compose.sh
@@ -63,7 +63,7 @@ source "${COMPOSE_FOLDER}"/get-versions.sh > /dev/null 2>&1
 
 # in case of csit running for PAP (groups should be for pap) but starts apex-pdp for dependencies.
 if [ -z "$PROJECT" ]; then
-  PROJECT=$component
+  export PROJECT=$component
 fi
 
 if [ -n "$component" ]; then
@@ -82,7 +82,7 @@ if [ -n "$component" ]; then
     docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml up -d "${component}"
   fi
 else
-  PROJECT=pap
+  export PROJECT=api # api has groups.json complete with all 3 pdps
   if [ "$gui" = true ]; then
     echo "Starting application with gui..."
     docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml \
diff --git a/csit/run-k8s-csit.sh b/csit/run-k8s-csit.sh
index 201c1f5a..fb52b79f 100755
--- a/csit/run-k8s-csit.sh
+++ b/csit/run-k8s-csit.sh
@@ -30,7 +30,7 @@ CSIT_SCRIPT="scripts/run-test.sh"
 ROBOT_DOCKER_IMAGE="policy-csit-robot"
 POLICY_CLAMP_ROBOT="policy-clamp-test.robot"
 POLICY_API_ROBOT="api-test.robot"
-POLICY_PAP_ROBOT="pap-test.robot"
+POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot"
 POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot"
 POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot"
 POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot"
@@ -87,6 +87,8 @@ function teardown_cluster () {
     sudo snap remove microk8s;rm -rf $HOME/.kube/config
     sudo rm -rf /dockerdata-nfs/mariadb-galera/
     echo "K8s Cluster removed"
+    echo "Clean up docker"
+    docker system prune -af
 }
 
 
@@ -107,16 +109,21 @@ function build_robot_image () {
     echo "Importing robot image into microk8s registry"
     docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest
     microk8s ctr image import policy-csit-robot.tar
+}
+
+
+function start_csit () {
+    build_robot_image
     if [ "${?}" -eq 0 ]; then
-        rm -rf policy-csit-robot.tar
-        rm -rf tests/models/
+        rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar
+        rm -rf ${WORKSPACE}/csit/resources/tests/models/
         echo "---------------------------------------------"
         echo "Installing Robot framework pod for running CSIT"
         cd ${WORKSPACE}/helm
         mkdir -p ${ROBOT_LOG_DIR}
         microk8s helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={${READINESS_CONTAINERS[*]}}" --set robotLogDir=$ROBOT_LOG_DIR;
         print_robot_log
-    fi
+  fi
 }
 
 
@@ -218,6 +225,7 @@ if [ $1 == "install" ];  then
         cd ${WORKSPACE}/helm || exit;
         microk8s helm dependency build policy
         microk8s helm install csit-policy policy
+        microk8s helm install prometheus prometheus
         echo "Policy chart installation completed"
 	    echo "-------------------------------------------"
     fi
@@ -229,7 +237,7 @@ if [ $1 == "install" ];  then
         echo "CSIT will be invoked from $ROBOT_FILE"
         echo "Readiness containers: ${READINESS_CONTAINERS[*]}"
         echo "-------------------------------------------"
-        build_robot_image
+        start_csit
     else
         echo "No project supplied for running CSIT"
     fi
diff --git a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml b/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml
deleted file mode 100644
index 3e7bad66..00000000
--- a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-{{/*
-# Copyright © 2022 Nordix Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/}}
-
-{{- if default false (and .Values.global.metrics.enabled .Values.global.metrics.custom_resources) }}
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: { .Chart.Name }}
-  namespace: {{ include "common.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ .Chart.Name }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    app.kubernetes.io/managed-by: Helm
-
-    {{- toYaml .Values.metrics.prometheusRules.selector | nindent 4 }}
-spec:
-  groups:
-  - name: {{ .Chart.Name }}
-    rules:
-      {{- toYaml .Values.metrics.prometheusRules.rules | nindent 6 }}
-{{- end }}
-
diff --git a/helm/policy/components/mariadb-galera/templates/statefulset.yaml b/helm/policy/components/mariadb-galera/templates/statefulset.yaml
index 3644a9bd..c347bef0 100644
--- a/helm/policy/components/mariadb-galera/templates/statefulset.yaml
+++ b/helm/policy/components/mariadb-galera/templates/statefulset.yaml
@@ -223,52 +223,6 @@ spec:
             - name: mariadb-galera-actual-config
               mountPath: /opt/bitnami/mariadb/conf
             {{- end }}
-        {{- if default false .Values.global.metrics.enabled }}
-        - name: {{ .Chart.Name }}-metrics
-          image: {{ .Values.metrics.image }}
-          imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.metrics.pullPolicy | quote}}
-          env:
-            - name: MARIADB_METRICS_EXTRA_FLAGS
-              value: {{ default "" (join " " .Values.metrics.extraFlags) | quote }}
-            - name: MARIADB_ROOT_USER
-              value: {{ .Values.rootUser.user | quote }}
-            - name: MARIADB_ROOT_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: mariadb-galera-db-root-password
-                  key: password
-          command:
-            - sh
-            - -c
-            - |
-              DATA_SOURCE_NAME="$MARIADB_ROOT_USER:$MARIADB_ROOT_PASSWORD@(localhost:3306)/" /bin/mysqld_exporter $MARIADB_METRICS_EXTRA_FLAGS
-          ports:
-            - name: tcp-metrics
-              containerPort: 9104
-          livenessProbe:
-            httpGet:
-              path: /metrics
-              port: tcp-metrics
-            initialDelaySeconds: {{ .Values.metrics.livenessProbe.initialDelaySeconds }}
-            periodSeconds: {{ .Values.metrics.livenessProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.metrics.livenessProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.metrics.livenessProbe.successThreshold }}
-            failureThreshold: {{ .Values.metrics.livenessProbe.failureThreshold }}
-          readinessProbe:
-            httpGet:
-              path: /metrics
-              port: tcp-metrics
-            initialDelaySeconds: {{ .Values.metrics.readinessProbe.initialDelaySeconds }}
-            periodSeconds: {{ .Values.metrics.readinessProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.metrics.readinessProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.metrics.readinessProbe.successThreshold }}
-            failureThreshold: {{ .Values.metrics.readinessProbe.failureThreshold }}
-          securityContext:
-            runAsUser: 10001
-            runAsGroup: 10001
-            fsGroup: 10001
-          resources: {{- toYaml .Values.metrics.resources | nindent 12 }}
-        {{- end }}
       imagePullSecrets:
         - name: default-docker-registry-key
       {{- if .Values.schedulerName }}
diff --git a/helm/prometheus/Chart.yaml b/helm/prometheus/Chart.yaml
new file mode 100644
index 00000000..75e5a2b7
--- /dev/null
+++ b/helm/prometheus/Chart.yaml
@@ -0,0 +1,27 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v2
+name: prometheus
+description: Prometheus server to collect all policy components metrics
+keywords:
+- prometheus
+- metrics
+- monitoring
+type: application
+version: 0.1.0
diff --git a/helm/prometheus/resources/prometheus.yml b/helm/prometheus/resources/prometheus.yml
new file mode 100644
index 00000000..70041c97
--- /dev/null
+++ b/helm/prometheus/resources/prometheus.yml
@@ -0,0 +1,96 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+# global config
+global:
+  scrape_interval: 60s
+  evaluation_interval: 10s
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# scrape config
+scrape_configs:
+  - job_name: "api-metrics"
+    metrics_path: /policy/api/v1/metrics
+    static_configs:
+      - targets: ["policy-api:6969"]
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "pap-metrics"
+    metrics_path: /policy/pap/v1/metrics
+    static_configs:
+      - targets: ["policy-pap:6969"]
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "apex-pdp-metrics"
+    static_configs:
+      - targets:
+          - "policy-apex-pdp:6969"
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  # - job_name: "drools-apps-metrics"
+  #   static_configs:
+  #     - targets:
+  #         - "drools-apps:9696"
+  #   basic_auth:
+  #     username: "demo@people.osaaf.org"
+  #     password: "demo123456!"
+
+  - job_name: "drools-pdp-metrics"
+    static_configs:
+      - targets:
+          - "policy-drools-pdp:9696"
+    basic_auth:
+      username: "demo@people.osaaf.org"
+      password: "demo123456!"
+
+  - job_name: "distribution-metrics"
+    static_configs:
+      - targets:
+          - "policy-distribution:6969"
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "xacml-pdp-metrics"
+    static_configs:
+      - targets:
+          - "policy-xacml-pdp:6969"
+    basic_auth:
+      username: "policyadmin"
+      password: "zb!XztG34"
+
+  - job_name: "acm-metrics"
+    metrics_path: "/onap/policy/clamp/acm/prometheus"
+    static_configs:
+      - targets:
+          - "policy-clamp-runtime-acm:6969"
+    basic_auth:
+      username: "runtimeUser"
+      password: "zb!XztG34"
diff --git a/helm/prometheus/templates/cluster-role.yaml b/helm/prometheus/templates/cluster-role.yaml
new file mode 100644
index 00000000..d8ce874c
--- /dev/null
+++ b/helm/prometheus/templates/cluster-role.yaml
@@ -0,0 +1,52 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Chart.Name }}
+  namespace: default
+rules:
+- apiGroups: [""]
+  resources:
+  - nodes
+  - services
+  - endpoints
+  - pods
+  verbs: ["get", "list", "watch"]
+- apiGroups:
+  - extensions
+  resources:
+  - ingresses
+  verbs: ["get", "list", "watch"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Chart.Name }}-read
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Chart.Name }}-read
+subjects:
+- kind: ServiceAccount
+  name: {{ .Chart.Name }}-read
+  namespace: default
diff --git a/helm/prometheus/templates/configmap.yaml b/helm/prometheus/templates/configmap.yaml
new file mode 100644
index 00000000..05c2964b
--- /dev/null
+++ b/helm/prometheus/templates/configmap.yaml
@@ -0,0 +1,35 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Chart.Name }}-configmap
+  namespace: default
+  labels:
+    app: {{ .Chart.Name }}
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    heritage: Helm
+{{- with .Files.Glob "resources/*store" }}
+binaryData:
+{{- range $path, $bytes := . }}
+  {{ base $path }}: {{ $.Files.Get $path | b64enc | quote }}
+{{- end }}
+{{- end }}
+data:
+{{ tpl (.Files.Glob "resources/*.{yaml,yml}").AsConfig . | indent 2 }}
\ No newline at end of file
diff --git a/helm/prometheus/templates/deployment.yaml b/helm/prometheus/templates/deployment.yaml
new file mode 100644
index 00000000..818af85f
--- /dev/null
+++ b/helm/prometheus/templates/deployment.yaml
@@ -0,0 +1,57 @@
+
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Chart.Name }}
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ .Chart.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ .Chart.Name }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+        app.kubernetes.io/instance: RELEASE
+        app.kubernetes.io/managed-by: Helm
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9090"
+    spec:
+      containers:
+        - image: prom/prometheus:v2.32.1
+          name: prometheus
+          args:
+          - '--config.file=/etc/prometheus/prometheus.yml'
+          ports:
+            - containerPort: 9090
+          resources: {}
+          volumeMounts:
+            - mountPath: /etc/prometheus
+              name: prometheusconfig
+      hostname: prometheus
+      restartPolicy: Always
+      volumes:
+        - name: prometheusconfig
+          configMap:
+              name: {{ .Chart.Name }}-configmap
+              defaultMode: 0755
diff --git a/helm/prometheus/templates/role-binding.yaml b/helm/prometheus/templates/role-binding.yaml
new file mode 100644
index 00000000..355e4eda
--- /dev/null
+++ b/helm/prometheus/templates/role-binding.yaml
@@ -0,0 +1,30 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Chart.Name }}-read
+  namespace: default
+subjects:
+- kind: ServiceAccount
+  name: {{ .Chart.Name }}-read
+roleRef:
+  kind: Role
+  name: read
+  apiGroup: rbac.authorization.k8s.io
diff --git a/helm/prometheus/templates/service-account.yaml b/helm/prometheus/templates/service-account.yaml
new file mode 100644
index 00000000..3f34b5b9
--- /dev/null
+++ b/helm/prometheus/templates/service-account.yaml
@@ -0,0 +1,23 @@
+#
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Chart.Name }}-read
+  namespace: default
diff --git a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml b/helm/prometheus/templates/service.yaml
index 00917452..2ce3b028 100644
--- a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml
+++ b/helm/prometheus/templates/service.yaml
@@ -1,37 +1,38 @@
-{{/*
-# Copyright © 2022 Nordix Foundation
 #
+# ===========LICENSE_START====================================================
+#  Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#       http://www.apache.org/licenses/LICENSE-2.0
+#      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-*/}}
+# ============LICENSE_END=====================================================
+#
 
-{{- if default false .Values.global.metrics.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ .Chart.Name }}-metrics
+  name: {{ .Chart.Name }}
   namespace: default
   labels:
     app.kubernetes.io/name: {{ .Chart.Name }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: Helm
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '9090'
 spec:
-  type: {{ .Values.metrics.service.type }}
+  type: NodePort
   ports:
-    - name: tcp-metrics
-      port: {{ .Values.metrics.service.port }}
-      targetPort: tcp-metrics
+    - port: 9090
+      targetPort: 9090
+      nodePort: 30909
   selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ .Chart.Name }}
-
-{{- end }}
+    app.kubernetes.io/name: {{ .Chart.Name }}