summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authoradheli.tavares <adheli.tavares@est.tech>2023-03-13 16:55:02 +0000
committerAdheli Tavares <adheli.tavares@est.tech>2023-03-14 15:10:40 +0000
commitc244ab0c6dfe6c9a4d23a35710659b511d6d74cb (patch)
treeb7c97e15da2a76ca95795aad0a3ee6e3211da30e
parent4bde5ee757d1141f2e1f1ef55f6e25774d6e806f (diff)
Add prometheus server to k8s charts
- add pap-slas.test to list of robot tests for k8s - fix an export var on docker compose Issue-ID: POLICY-4545 Change-Id: Id1dd55693ab6d016fd0dd38230972ffe96b28b14 Signed-off-by: adheli.tavares <adheli.tavares@est.tech>
-rwxr-xr-xcompose/start-compose.sh4
-rwxr-xr-xcsit/run-k8s-csit.sh18
-rw-r--r--helm/policy/components/mariadb-galera/templates/prometheusrules.yaml35
-rw-r--r--helm/policy/components/mariadb-galera/templates/statefulset.yaml46
-rw-r--r--helm/prometheus/Chart.yaml27
-rw-r--r--helm/prometheus/resources/prometheus.yml96
-rw-r--r--helm/prometheus/templates/cluster-role.yaml52
-rw-r--r--helm/prometheus/templates/configmap.yaml35
-rw-r--r--helm/prometheus/templates/deployment.yaml57
-rw-r--r--helm/prometheus/templates/role-binding.yaml30
-rw-r--r--helm/prometheus/templates/service-account.yaml23
-rw-r--r--helm/prometheus/templates/service.yaml (renamed from helm/policy/components/mariadb-galera/templates/metrics-svc.yaml)29
12 files changed, 350 insertions, 102 deletions
diff --git a/compose/start-compose.sh b/compose/start-compose.sh
index 57b55316..a77b4834 100755
--- a/compose/start-compose.sh
+++ b/compose/start-compose.sh
@@ -63,7 +63,7 @@ source "${COMPOSE_FOLDER}"/get-versions.sh > /dev/null 2>&1
# in case of csit running for PAP (groups should be for pap) but starts apex-pdp for dependencies.
if [ -z "$PROJECT" ]; then
- PROJECT=$component
+ export PROJECT=$component
fi
if [ -n "$component" ]; then
@@ -82,7 +82,7 @@ if [ -n "$component" ]; then
docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml up -d "${component}"
fi
else
- PROJECT=pap
+ export PROJECT=api # api has groups.json complete with all 3 pdps
if [ "$gui" = true ]; then
echo "Starting application with gui..."
docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml \
diff --git a/csit/run-k8s-csit.sh b/csit/run-k8s-csit.sh
index 201c1f5a..fb52b79f 100755
--- a/csit/run-k8s-csit.sh
+++ b/csit/run-k8s-csit.sh
@@ -30,7 +30,7 @@ CSIT_SCRIPT="scripts/run-test.sh"
ROBOT_DOCKER_IMAGE="policy-csit-robot"
POLICY_CLAMP_ROBOT="policy-clamp-test.robot"
POLICY_API_ROBOT="api-test.robot"
-POLICY_PAP_ROBOT="pap-test.robot"
+POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot"
POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot"
POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot"
POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot"
@@ -87,6 +87,8 @@ function teardown_cluster () {
sudo snap remove microk8s;rm -rf $HOME/.kube/config
sudo rm -rf /dockerdata-nfs/mariadb-galera/
echo "K8s Cluster removed"
+ echo "Clean up docker"
+ docker system prune -af
}
@@ -107,16 +109,21 @@ function build_robot_image () {
echo "Importing robot image into microk8s registry"
docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest
microk8s ctr image import policy-csit-robot.tar
+}
+
+
+function start_csit () {
+ build_robot_image
if [ "${?}" -eq 0 ]; then
- rm -rf policy-csit-robot.tar
- rm -rf tests/models/
+ rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar
+ rm -rf ${WORKSPACE}/csit/resources/tests/models/
echo "---------------------------------------------"
echo "Installing Robot framework pod for running CSIT"
cd ${WORKSPACE}/helm
mkdir -p ${ROBOT_LOG_DIR}
microk8s helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={${READINESS_CONTAINERS[*]}}" --set robotLogDir=$ROBOT_LOG_DIR;
print_robot_log
- fi
+ fi
}
@@ -218,6 +225,7 @@ if [ $1 == "install" ]; then
cd ${WORKSPACE}/helm || exit;
microk8s helm dependency build policy
microk8s helm install csit-policy policy
+ microk8s helm install prometheus prometheus
echo "Policy chart installation completed"
echo "-------------------------------------------"
fi
@@ -229,7 +237,7 @@ if [ $1 == "install" ]; then
echo "CSIT will be invoked from $ROBOT_FILE"
echo "Readiness containers: ${READINESS_CONTAINERS[*]}"
echo "-------------------------------------------"
- build_robot_image
+ start_csit
else
echo "No project supplied for running CSIT"
fi
diff --git a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml b/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml
deleted file mode 100644
index 3e7bad66..00000000
--- a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
-{{/*
-# Copyright © 2022 Nordix Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/}}
-
-{{- if default false (and .Values.global.metrics.enabled .Values.global.metrics.custom_resources) }}
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
- name: { .Chart.Name }}
- namespace: {{ include "common.namespace" . }}
- labels:
- app.kubernetes.io/name: {{ .Chart.Name }}
- helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
- app.kubernetes.io/managed-by: Helm
-
- {{- toYaml .Values.metrics.prometheusRules.selector | nindent 4 }}
-spec:
- groups:
- - name: {{ .Chart.Name }}
- rules:
- {{- toYaml .Values.metrics.prometheusRules.rules | nindent 6 }}
-{{- end }}
-
diff --git a/helm/policy/components/mariadb-galera/templates/statefulset.yaml b/helm/policy/components/mariadb-galera/templates/statefulset.yaml
index 3644a9bd..c347bef0 100644
--- a/helm/policy/components/mariadb-galera/templates/statefulset.yaml
+++ b/helm/policy/components/mariadb-galera/templates/statefulset.yaml
@@ -223,52 +223,6 @@ spec:
- name: mariadb-galera-actual-config
mountPath: /opt/bitnami/mariadb/conf
{{- end }}
- {{- if default false .Values.global.metrics.enabled }}
- - name: {{ .Chart.Name }}-metrics
- image: {{ .Values.metrics.image }}
- imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.metrics.pullPolicy | quote}}
- env:
- - name: MARIADB_METRICS_EXTRA_FLAGS
- value: {{ default "" (join " " .Values.metrics.extraFlags) | quote }}
- - name: MARIADB_ROOT_USER
- value: {{ .Values.rootUser.user | quote }}
- - name: MARIADB_ROOT_PASSWORD
- valueFrom:
- secretKeyRef:
- name: mariadb-galera-db-root-password
- key: password
- command:
- - sh
- - -c
- - |
- DATA_SOURCE_NAME="$MARIADB_ROOT_USER:$MARIADB_ROOT_PASSWORD@(localhost:3306)/" /bin/mysqld_exporter $MARIADB_METRICS_EXTRA_FLAGS
- ports:
- - name: tcp-metrics
- containerPort: 9104
- livenessProbe:
- httpGet:
- path: /metrics
- port: tcp-metrics
- initialDelaySeconds: {{ .Values.metrics.livenessProbe.initialDelaySeconds }}
- periodSeconds: {{ .Values.metrics.livenessProbe.periodSeconds }}
- timeoutSeconds: {{ .Values.metrics.livenessProbe.timeoutSeconds }}
- successThreshold: {{ .Values.metrics.livenessProbe.successThreshold }}
- failureThreshold: {{ .Values.metrics.livenessProbe.failureThreshold }}
- readinessProbe:
- httpGet:
- path: /metrics
- port: tcp-metrics
- initialDelaySeconds: {{ .Values.metrics.readinessProbe.initialDelaySeconds }}
- periodSeconds: {{ .Values.metrics.readinessProbe.periodSeconds }}
- timeoutSeconds: {{ .Values.metrics.readinessProbe.timeoutSeconds }}
- successThreshold: {{ .Values.metrics.readinessProbe.successThreshold }}
- failureThreshold: {{ .Values.metrics.readinessProbe.failureThreshold }}
- securityContext:
- runAsUser: 10001
- runAsGroup: 10001
- fsGroup: 10001
- resources: {{- toYaml .Values.metrics.resources | nindent 12 }}
- {{- end }}
imagePullSecrets:
- name: default-docker-registry-key
{{- if .Values.schedulerName }}
diff --git a/helm/prometheus/Chart.yaml b/helm/prometheus/Chart.yaml
new file mode 100644
index 00000000..75e5a2b7
--- /dev/null
+++ b/helm/prometheus/Chart.yaml
@@ -0,0 +1,27 @@
+#
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v2
+name: prometheus
+description: Prometheus server to collect all policy components metrics
+keywords:
+- prometheus
+- metrics
+- monitoring
+type: application
+version: 0.1.0
diff --git a/helm/prometheus/resources/prometheus.yml b/helm/prometheus/resources/prometheus.yml
new file mode 100644
index 00000000..70041c97
--- /dev/null
+++ b/helm/prometheus/resources/prometheus.yml
@@ -0,0 +1,96 @@
+#
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+# global config
+global:
+ scrape_interval: 60s
+ evaluation_interval: 10s
+
+# Alertmanager configuration
+alerting:
+ alertmanagers:
+ - static_configs:
+ - targets:
+ # - alertmanager:9093
+
+# scrape config
+scrape_configs:
+ - job_name: "api-metrics"
+ metrics_path: /policy/api/v1/metrics
+ static_configs:
+ - targets: ["policy-api:6969"]
+ basic_auth:
+ username: "policyadmin"
+ password: "zb!XztG34"
+
+ - job_name: "pap-metrics"
+ metrics_path: /policy/pap/v1/metrics
+ static_configs:
+ - targets: ["policy-pap:6969"]
+ basic_auth:
+ username: "policyadmin"
+ password: "zb!XztG34"
+
+ - job_name: "apex-pdp-metrics"
+ static_configs:
+ - targets:
+ - "policy-apex-pdp:6969"
+ basic_auth:
+ username: "policyadmin"
+ password: "zb!XztG34"
+
+ # - job_name: "drools-apps-metrics"
+ # static_configs:
+ # - targets:
+ # - "drools-apps:9696"
+ # basic_auth:
+ # username: "demo@people.osaaf.org"
+ # password: "demo123456!"
+
+ - job_name: "drools-pdp-metrics"
+ static_configs:
+ - targets:
+ - "policy-drools-pdp:9696"
+ basic_auth:
+ username: "demo@people.osaaf.org"
+ password: "demo123456!"
+
+ - job_name: "distribution-metrics"
+ static_configs:
+ - targets:
+ - "policy-distribution:6969"
+ basic_auth:
+ username: "policyadmin"
+ password: "zb!XztG34"
+
+ - job_name: "xacml-pdp-metrics"
+ static_configs:
+ - targets:
+ - "policy-xacml-pdp:6969"
+ basic_auth:
+ username: "policyadmin"
+ password: "zb!XztG34"
+
+ - job_name: "acm-metrics"
+ metrics_path: "/onap/policy/clamp/acm/prometheus"
+ static_configs:
+ - targets:
+ - "policy-clamp-runtime-acm:6969"
+ basic_auth:
+ username: "runtimeUser"
+ password: "zb!XztG34"
diff --git a/helm/prometheus/templates/cluster-role.yaml b/helm/prometheus/templates/cluster-role.yaml
new file mode 100644
index 00000000..d8ce874c
--- /dev/null
+++ b/helm/prometheus/templates/cluster-role.yaml
@@ -0,0 +1,52 @@
+#
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: {{ .Chart.Name }}
+ namespace: default
+rules:
+- apiGroups: [""]
+ resources:
+ - nodes
+ - services
+ - endpoints
+ - pods
+ verbs: ["get", "list", "watch"]
+- apiGroups:
+ - extensions
+ resources:
+ - ingresses
+ verbs: ["get", "list", "watch"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: {{ .Chart.Name }}-read
+ namespace: default
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: {{ .Chart.Name }}-read
+subjects:
+- kind: ServiceAccount
+ name: {{ .Chart.Name }}-read
+ namespace: default
diff --git a/helm/prometheus/templates/configmap.yaml b/helm/prometheus/templates/configmap.yaml
new file mode 100644
index 00000000..05c2964b
--- /dev/null
+++ b/helm/prometheus/templates/configmap.yaml
@@ -0,0 +1,35 @@
+#
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ .Chart.Name }}-configmap
+ namespace: default
+ labels:
+ app: {{ .Chart.Name }}
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ heritage: Helm
+{{- with .Files.Glob "resources/*store" }}
+binaryData:
+{{- range $path, $bytes := . }}
+ {{ base $path }}: {{ $.Files.Get $path | b64enc | quote }}
+{{- end }}
+{{- end }}
+data:
+{{ tpl (.Files.Glob "resources/*.{yaml,yml}").AsConfig . | indent 2 }} \ No newline at end of file
diff --git a/helm/prometheus/templates/deployment.yaml b/helm/prometheus/templates/deployment.yaml
new file mode 100644
index 00000000..818af85f
--- /dev/null
+++ b/helm/prometheus/templates/deployment.yaml
@@ -0,0 +1,57 @@
+
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: {{ .Chart.Name }}
+ namespace: default
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app.kubernetes.io/name: {{ .Chart.Name }}
+ template:
+ metadata:
+ labels:
+ app.kubernetes.io/name: {{ .Chart.Name }}
+ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ app.kubernetes.io/instance: RELEASE
+ app.kubernetes.io/managed-by: Helm
+ annotations:
+ prometheus.io/scrape: "true"
+ prometheus.io/port: "9090"
+ spec:
+ containers:
+ - image: prom/prometheus:v2.32.1
+ name: prometheus
+ args:
+ - '--config.file=/etc/prometheus/prometheus.yml'
+ ports:
+ - containerPort: 9090
+ resources: {}
+ volumeMounts:
+ - mountPath: /etc/prometheus
+ name: prometheusconfig
+ hostname: prometheus
+ restartPolicy: Always
+ volumes:
+ - name: prometheusconfig
+ configMap:
+ name: {{ .Chart.Name }}-configmap
+ defaultMode: 0755
diff --git a/helm/prometheus/templates/role-binding.yaml b/helm/prometheus/templates/role-binding.yaml
new file mode 100644
index 00000000..355e4eda
--- /dev/null
+++ b/helm/prometheus/templates/role-binding.yaml
@@ -0,0 +1,30 @@
+#
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: {{ .Chart.Name }}-read
+ namespace: default
+subjects:
+- kind: ServiceAccount
+ name: {{ .Chart.Name }}-read
+roleRef:
+ kind: Role
+ name: read
+ apiGroup: rbac.authorization.k8s.io
diff --git a/helm/prometheus/templates/service-account.yaml b/helm/prometheus/templates/service-account.yaml
new file mode 100644
index 00000000..3f34b5b9
--- /dev/null
+++ b/helm/prometheus/templates/service-account.yaml
@@ -0,0 +1,23 @@
+#
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============LICENSE_END=====================================================
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: {{ .Chart.Name }}-read
+ namespace: default
diff --git a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml b/helm/prometheus/templates/service.yaml
index 00917452..2ce3b028 100644
--- a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml
+++ b/helm/prometheus/templates/service.yaml
@@ -1,37 +1,38 @@
-{{/*
-# Copyright © 2022 Nordix Foundation
#
+# ===========LICENSE_START====================================================
+# Copyright (C) 2023 Nordix Foundation.
+# ============================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-*/}}
+# ============LICENSE_END=====================================================
+#
-{{- if default false .Values.global.metrics.enabled }}
apiVersion: v1
kind: Service
metadata:
- name: {{ .Chart.Name }}-metrics
+ name: {{ .Chart.Name }}
namespace: default
labels:
app.kubernetes.io/name: {{ .Chart.Name }}
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
app.kubernetes.io/managed-by: Helm
+ annotations:
+ prometheus.io/scrape: 'true'
+ prometheus.io/port: '9090'
spec:
- type: {{ .Values.metrics.service.type }}
+ type: NodePort
ports:
- - name: tcp-metrics
- port: {{ .Values.metrics.service.port }}
- targetPort: tcp-metrics
+ - port: 9090
+ targetPort: 9090
+ nodePort: 30909
selector:
- matchLabels:
- app.kubernetes.io/name: {{ .Chart.Name }}
-
-{{- end }}
+ app.kubernetes.io/name: {{ .Chart.Name }}