diff options
| -rwxr-xr-x | compose/start-compose.sh | 4 | ||||
| -rwxr-xr-x | csit/run-k8s-csit.sh | 18 | ||||
| -rw-r--r-- | helm/policy/components/mariadb-galera/templates/prometheusrules.yaml | 35 | ||||
| -rw-r--r-- | helm/policy/components/mariadb-galera/templates/statefulset.yaml | 46 | ||||
| -rw-r--r-- | helm/prometheus/Chart.yaml | 27 | ||||
| -rw-r--r-- | helm/prometheus/resources/prometheus.yml | 96 | ||||
| -rw-r--r-- | helm/prometheus/templates/cluster-role.yaml | 52 | ||||
| -rw-r--r-- | helm/prometheus/templates/configmap.yaml | 35 | ||||
| -rw-r--r-- | helm/prometheus/templates/deployment.yaml | 57 | ||||
| -rw-r--r-- | helm/prometheus/templates/role-binding.yaml | 30 | ||||
| -rw-r--r-- | helm/prometheus/templates/service-account.yaml | 23 | ||||
| -rw-r--r-- | helm/prometheus/templates/service.yaml (renamed from helm/policy/components/mariadb-galera/templates/metrics-svc.yaml) | 29 |
12 files changed, 350 insertions, 102 deletions
diff --git a/compose/start-compose.sh b/compose/start-compose.sh index 57b55316..a77b4834 100755 --- a/compose/start-compose.sh +++ b/compose/start-compose.sh @@ -63,7 +63,7 @@ source "${COMPOSE_FOLDER}"/get-versions.sh > /dev/null 2>&1 # in case of csit running for PAP (groups should be for pap) but starts apex-pdp for dependencies. if [ -z "$PROJECT" ]; then - PROJECT=$component + export PROJECT=$component fi if [ -n "$component" ]; then @@ -82,7 +82,7 @@ if [ -n "$component" ]; then docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml up -d "${component}" fi else - PROJECT=pap + export PROJECT=api # api has groups.json complete with all 3 pdps if [ "$gui" = true ]; then echo "Starting application with gui..." docker-compose -f "${COMPOSE_FOLDER}"/docker-compose.yml \ diff --git a/csit/run-k8s-csit.sh b/csit/run-k8s-csit.sh index 201c1f5a..fb52b79f 100755 --- a/csit/run-k8s-csit.sh +++ b/csit/run-k8s-csit.sh @@ -30,7 +30,7 @@ CSIT_SCRIPT="scripts/run-test.sh" ROBOT_DOCKER_IMAGE="policy-csit-robot" POLICY_CLAMP_ROBOT="policy-clamp-test.robot" POLICY_API_ROBOT="api-test.robot" -POLICY_PAP_ROBOT="pap-test.robot" +POLICY_PAP_ROBOT="pap-test.robot pap-slas.robot" POLICY_APEX_PDP_ROBOT="apex-pdp-test.robot" POLICY_XACML_PDP_ROBOT="xacml-pdp-test.robot" POLICY_DROOLS_PDP_ROBOT="drools-pdp-test.robot" @@ -87,6 +87,8 @@ function teardown_cluster () { sudo snap remove microk8s;rm -rf $HOME/.kube/config sudo rm -rf /dockerdata-nfs/mariadb-galera/ echo "K8s Cluster removed" + echo "Clean up docker" + docker system prune -af } @@ -107,16 +109,21 @@ function build_robot_image () { echo "Importing robot image into microk8s registry" docker save -o policy-csit-robot.tar ${ROBOT_DOCKER_IMAGE}:latest microk8s ctr image import policy-csit-robot.tar +} + + +function start_csit () { + build_robot_image if [ "${?}" -eq 0 ]; then - rm -rf policy-csit-robot.tar - rm -rf tests/models/ + rm -rf ${WORKSPACE}/csit/resources/policy-csit-robot.tar + rm -rf ${WORKSPACE}/csit/resources/tests/models/ echo "---------------------------------------------" echo "Installing Robot framework pod for running CSIT" cd ${WORKSPACE}/helm mkdir -p ${ROBOT_LOG_DIR} microk8s helm install csit-robot robot --set robot="$ROBOT_FILE" --set "readiness={${READINESS_CONTAINERS[*]}}" --set robotLogDir=$ROBOT_LOG_DIR; print_robot_log - fi + fi } @@ -218,6 +225,7 @@ if [ $1 == "install" ]; then cd ${WORKSPACE}/helm || exit; microk8s helm dependency build policy microk8s helm install csit-policy policy + microk8s helm install prometheus prometheus echo "Policy chart installation completed" echo "-------------------------------------------" fi @@ -229,7 +237,7 @@ if [ $1 == "install" ]; then echo "CSIT will be invoked from $ROBOT_FILE" echo "Readiness containers: ${READINESS_CONTAINERS[*]}" echo "-------------------------------------------" - build_robot_image + start_csit else echo "No project supplied for running CSIT" fi diff --git a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml b/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml deleted file mode 100644 index 3e7bad66..00000000 --- a/helm/policy/components/mariadb-galera/templates/prometheusrules.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{/* -# Copyright © 2022 Nordix Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -*/}} - -{{- if default false (and .Values.global.metrics.enabled .Values.global.metrics.custom_resources) }} -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: { .Chart.Name }} - namespace: {{ include "common.namespace" . }} - labels: - app.kubernetes.io/name: {{ .Chart.Name }} - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - app.kubernetes.io/managed-by: Helm - - {{- toYaml .Values.metrics.prometheusRules.selector | nindent 4 }} -spec: - groups: - - name: {{ .Chart.Name }} - rules: - {{- toYaml .Values.metrics.prometheusRules.rules | nindent 6 }} -{{- end }} - diff --git a/helm/policy/components/mariadb-galera/templates/statefulset.yaml b/helm/policy/components/mariadb-galera/templates/statefulset.yaml index 3644a9bd..c347bef0 100644 --- a/helm/policy/components/mariadb-galera/templates/statefulset.yaml +++ b/helm/policy/components/mariadb-galera/templates/statefulset.yaml @@ -223,52 +223,6 @@ spec: - name: mariadb-galera-actual-config mountPath: /opt/bitnami/mariadb/conf {{- end }} - {{- if default false .Values.global.metrics.enabled }} - - name: {{ .Chart.Name }}-metrics - image: {{ .Values.metrics.image }} - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.metrics.pullPolicy | quote}} - env: - - name: MARIADB_METRICS_EXTRA_FLAGS - value: {{ default "" (join " " .Values.metrics.extraFlags) | quote }} - - name: MARIADB_ROOT_USER - value: {{ .Values.rootUser.user | quote }} - - name: MARIADB_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: mariadb-galera-db-root-password - key: password - command: - - sh - - -c - - | - DATA_SOURCE_NAME="$MARIADB_ROOT_USER:$MARIADB_ROOT_PASSWORD@(localhost:3306)/" /bin/mysqld_exporter $MARIADB_METRICS_EXTRA_FLAGS - ports: - - name: tcp-metrics - containerPort: 9104 - livenessProbe: - httpGet: - path: /metrics - port: tcp-metrics - initialDelaySeconds: {{ .Values.metrics.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.metrics.livenessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.metrics.livenessProbe.timeoutSeconds }} - successThreshold: {{ .Values.metrics.livenessProbe.successThreshold }} - failureThreshold: {{ .Values.metrics.livenessProbe.failureThreshold }} - readinessProbe: - httpGet: - path: /metrics - port: tcp-metrics - initialDelaySeconds: {{ .Values.metrics.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.metrics.readinessProbe.periodSeconds }} - timeoutSeconds: {{ .Values.metrics.readinessProbe.timeoutSeconds }} - successThreshold: {{ .Values.metrics.readinessProbe.successThreshold }} - failureThreshold: {{ .Values.metrics.readinessProbe.failureThreshold }} - securityContext: - runAsUser: 10001 - runAsGroup: 10001 - fsGroup: 10001 - resources: {{- toYaml .Values.metrics.resources | nindent 12 }} - {{- end }} imagePullSecrets: - name: default-docker-registry-key {{- if .Values.schedulerName }} diff --git a/helm/prometheus/Chart.yaml b/helm/prometheus/Chart.yaml new file mode 100644 index 00000000..75e5a2b7 --- /dev/null +++ b/helm/prometheus/Chart.yaml @@ -0,0 +1,27 @@ +# +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END===================================================== +# + +apiVersion: v2 +name: prometheus +description: Prometheus server to collect all policy components metrics +keywords: +- prometheus +- metrics +- monitoring +type: application +version: 0.1.0 diff --git a/helm/prometheus/resources/prometheus.yml b/helm/prometheus/resources/prometheus.yml new file mode 100644 index 00000000..70041c97 --- /dev/null +++ b/helm/prometheus/resources/prometheus.yml @@ -0,0 +1,96 @@ +# +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END===================================================== +# + +# global config +global: + scrape_interval: 60s + evaluation_interval: 10s + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# scrape config +scrape_configs: + - job_name: "api-metrics" + metrics_path: /policy/api/v1/metrics + static_configs: + - targets: ["policy-api:6969"] + basic_auth: + username: "policyadmin" + password: "zb!XztG34" + + - job_name: "pap-metrics" + metrics_path: /policy/pap/v1/metrics + static_configs: + - targets: ["policy-pap:6969"] + basic_auth: + username: "policyadmin" + password: "zb!XztG34" + + - job_name: "apex-pdp-metrics" + static_configs: + - targets: + - "policy-apex-pdp:6969" + basic_auth: + username: "policyadmin" + password: "zb!XztG34" + + # - job_name: "drools-apps-metrics" + # static_configs: + # - targets: + # - "drools-apps:9696" + # basic_auth: + # username: "demo@people.osaaf.org" + # password: "demo123456!" + + - job_name: "drools-pdp-metrics" + static_configs: + - targets: + - "policy-drools-pdp:9696" + basic_auth: + username: "demo@people.osaaf.org" + password: "demo123456!" + + - job_name: "distribution-metrics" + static_configs: + - targets: + - "policy-distribution:6969" + basic_auth: + username: "policyadmin" + password: "zb!XztG34" + + - job_name: "xacml-pdp-metrics" + static_configs: + - targets: + - "policy-xacml-pdp:6969" + basic_auth: + username: "policyadmin" + password: "zb!XztG34" + + - job_name: "acm-metrics" + metrics_path: "/onap/policy/clamp/acm/prometheus" + static_configs: + - targets: + - "policy-clamp-runtime-acm:6969" + basic_auth: + username: "runtimeUser" + password: "zb!XztG34" diff --git a/helm/prometheus/templates/cluster-role.yaml b/helm/prometheus/templates/cluster-role.yaml new file mode 100644 index 00000000..d8ce874c --- /dev/null +++ b/helm/prometheus/templates/cluster-role.yaml @@ -0,0 +1,52 @@ +# +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END===================================================== +# + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Chart.Name }} + namespace: default +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Chart.Name }}-read + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Chart.Name }}-read +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-read + namespace: default diff --git a/helm/prometheus/templates/configmap.yaml b/helm/prometheus/templates/configmap.yaml new file mode 100644 index 00000000..05c2964b --- /dev/null +++ b/helm/prometheus/templates/configmap.yaml @@ -0,0 +1,35 @@ +# +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END===================================================== +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Chart.Name }}-configmap + namespace: default + labels: + app: {{ .Chart.Name }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + heritage: Helm +{{- with .Files.Glob "resources/*store" }} +binaryData: +{{- range $path, $bytes := . }} + {{ base $path }}: {{ $.Files.Get $path | b64enc | quote }} +{{- end }} +{{- end }} +data: +{{ tpl (.Files.Glob "resources/*.{yaml,yml}").AsConfig . | indent 2 }}
\ No newline at end of file diff --git a/helm/prometheus/templates/deployment.yaml b/helm/prometheus/templates/deployment.yaml new file mode 100644 index 00000000..818af85f --- /dev/null +++ b/helm/prometheus/templates/deployment.yaml @@ -0,0 +1,57 @@ + +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END===================================================== +# + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: {{ .Chart.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Chart.Name }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/instance: RELEASE + app.kubernetes.io/managed-by: Helm + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + containers: + - image: prom/prometheus:v2.32.1 + name: prometheus + args: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - containerPort: 9090 + resources: {} + volumeMounts: + - mountPath: /etc/prometheus + name: prometheusconfig + hostname: prometheus + restartPolicy: Always + volumes: + - name: prometheusconfig + configMap: + name: {{ .Chart.Name }}-configmap + defaultMode: 0755 diff --git a/helm/prometheus/templates/role-binding.yaml b/helm/prometheus/templates/role-binding.yaml new file mode 100644 index 00000000..355e4eda --- /dev/null +++ b/helm/prometheus/templates/role-binding.yaml @@ -0,0 +1,30 @@ +# +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END===================================================== +# + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Chart.Name }}-read + namespace: default +subjects: +- kind: ServiceAccount + name: {{ .Chart.Name }}-read +roleRef: + kind: Role + name: read + apiGroup: rbac.authorization.k8s.io diff --git a/helm/prometheus/templates/service-account.yaml b/helm/prometheus/templates/service-account.yaml new file mode 100644 index 00000000..3f34b5b9 --- /dev/null +++ b/helm/prometheus/templates/service-account.yaml @@ -0,0 +1,23 @@ +# +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END===================================================== +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Chart.Name }}-read + namespace: default diff --git a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml b/helm/prometheus/templates/service.yaml index 00917452..2ce3b028 100644 --- a/helm/policy/components/mariadb-galera/templates/metrics-svc.yaml +++ b/helm/prometheus/templates/service.yaml @@ -1,37 +1,38 @@ -{{/* -# Copyright © 2022 Nordix Foundation # +# ===========LICENSE_START==================================================== +# Copyright (C) 2023 Nordix Foundation. +# ============================================================================ # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -*/}} +# ============LICENSE_END===================================================== +# -{{- if default false .Values.global.metrics.enabled }} apiVersion: v1 kind: Service metadata: - name: {{ .Chart.Name }}-metrics + name: {{ .Chart.Name }} namespace: default labels: app.kubernetes.io/name: {{ .Chart.Name }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} app.kubernetes.io/managed-by: Helm + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9090' spec: - type: {{ .Values.metrics.service.type }} + type: NodePort ports: - - name: tcp-metrics - port: {{ .Values.metrics.service.port }} - targetPort: tcp-metrics + - port: 9090 + targetPort: 9090 + nodePort: 30909 selector: - matchLabels: - app.kubernetes.io/name: {{ .Chart.Name }} - -{{- end }} + app.kubernetes.io/name: {{ .Chart.Name }} |