From c50f0892fc601eb3d0e237c04b3f54019de513b1 Mon Sep 17 00:00:00 2001 From: Andreas Geissler Date: Wed, 14 Jun 2023 14:21:31 +0200 Subject: [POLICY] Correct clamp timeouts and enable sidecars in jobs again Clamp pods take longer to start in "small" flavor case and require 60 seconds to startup. Revert the Istio Sidecar removal in MariaDB jobs, as they are not the root cause of the startup issue. Issue-ID: OOM-3186 Signed-off-by: Andreas Geissler Change-Id: I0f3fd6a55e851640617bc3b0de8f96a0fe33d765 --- .../policy-clamp-ac-a1pms-ppnt/values.yaml | 4 +- .../policy-clamp-ac-http-ppnt/values.yaml | 4 +- .../policy-clamp-ac-k8s-ppnt/values.yaml | 4 +- .../policy-clamp-ac-kserve-ppnt/values.yaml | 4 +- .../components/policy-clamp-ac-pf-ppnt/values.yaml | 4 +- kubernetes/policy/templates/job.yaml | 84 +++++++++++++++++++--- 6 files changed, 86 insertions(+), 18 deletions(-) diff --git a/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml index 3e1fdf4dff..20ea23c4ab 100755 --- a/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml +++ b/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml @@ -81,7 +81,7 @@ serviceMesh: # probe configuration parameters liveness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container @@ -89,7 +89,7 @@ liveness: port: http-a1pms-api readiness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 port: http-a1pms-api diff --git a/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml index aeaf458daa..4842503fcc 100644 --- a/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml +++ b/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml @@ -71,7 +71,7 @@ serviceMesh: # probe configuration parameters liveness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container @@ -79,7 +79,7 @@ liveness: port: http-api readiness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 port: http-api diff --git a/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml index 0bbdf1307f..221d1405e1 100644 --- a/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml +++ b/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml @@ -64,7 +64,7 @@ nodeSelector: {} affinity: {} # probe configuration parameters liveness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container @@ -72,7 +72,7 @@ liveness: port: http-api readiness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 port: http-api diff --git a/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml index a793d615c2..237b946557 100755 --- a/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml +++ b/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml @@ -71,7 +71,7 @@ serviceMesh: # probe configuration parameters liveness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container @@ -79,7 +79,7 @@ liveness: port: kserve-api readiness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 port: kserve-api diff --git a/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml index a93fd866f9..c8387adc70 100644 --- a/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml +++ b/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml @@ -91,7 +91,7 @@ serviceMesh: # probe configuration parameters liveness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container @@ -99,7 +99,7 @@ liveness: port: http-api readiness: - initialDelaySeconds: 20 + initialDelaySeconds: 60 periodSeconds: 10 port: http-api diff --git a/kubernetes/policy/templates/job.yaml b/kubernetes/policy/templates/job.yaml index f0e91e8350..2503c6fd5f 100755 --- a/kubernetes/policy/templates/job.yaml +++ b/kubernetes/policy/templates/job.yaml @@ -32,8 +32,6 @@ spec: app: {{ include "common.name" . }}-galera-init release: {{ include "common.release" . }} name: {{ include "common.name" . }}-galera-init - annotations: - sidecar.istio.io/inject: "false" spec: imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" @@ -63,6 +61,8 @@ spec: - /bin/sh - -cx - | + {{- if include "common.onServiceMesh" . }} + echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }} /dbcmd-config/db.sh env: - name: MYSQL_ROOT_PASSWORD @@ -74,6 +74,23 @@ spec: - name: MYSQL_PORT value: "{{ index .Values "mariadb-galera" "service" "internalPort" }}" resources: {{ include "common.resources" . | nindent 10 }} + {{- if (include "common.onServiceMesh" .) }} + - name: policy-service-mesh-wait-for-job-container + image: {{ include "repositoryGenerator.image.quitQuit" . }} + imagePullPolicy: Always + command: + - /bin/sh + - "-c" + args: + - echo "waiting 10s for istio side cars to be up"; sleep 10s; + /app/ready.py --service-mesh-check {{ include "common.name" . }}-galera-config -t 45; + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- end }} restartPolicy: Never serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}} volumes: @@ -103,8 +120,6 @@ spec: app: {{ include "common.name" . }}-pg-init release: {{ include "common.release" . }} name: {{ include "common.name" . }}-pg-init - annotations: - sidecar.istio.io/inject: "false" spec: imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" @@ -121,6 +136,8 @@ spec: - /bin/sh - -cx - | + {{- if include "common.onServiceMesh" . }} + echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }} /docker-entrypoint-initdb.d/db-pg.sh env: - name: PG_ADMIN_PASSWORD @@ -134,6 +151,23 @@ spec: - name: PG_PORT value: "{{ .Values.postgres.service.internalPort }}" resources: {{ include "common.resources" . | nindent 10 }} + {{- if (include "common.onServiceMesh" .) }} + - name: policy-service-mesh-wait-for-job-container + image: {{ include "repositoryGenerator.image.quitQuit" . }} + imagePullPolicy: Always + command: + - /bin/sh + - "-c" + args: + - echo "waiting 10s for istio side cars to be up"; sleep 10s; + /app/ready.py --service-mesh-check {{ include "common.name" . }}-pg-config -t 45; + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- end }} restartPolicy: Never serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}} volumes: @@ -163,8 +197,6 @@ spec: app: {{ include "common.name" . }}-galera-config release: {{ include "common.release" . }} name: {{ include "common.name" . }}-galera-config - annotations: - sidecar.istio.io/inject: "false" spec: imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" @@ -195,6 +227,8 @@ spec: - /bin/sh - -cx - | + {{- if include "common.onServiceMesh" . }} + echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }} /dbcmd-config/db_migrator_policy_init.sh env: - name: SQL_HOST @@ -210,6 +244,23 @@ spec: - name: SCRIPT_DIRECTORY value: "sql" resources: {{ include "common.resources" . | nindent 10 }} + {{- if (include "common.onServiceMesh" .) }} + - name: policy-service-mesh-wait-for-job-container + image: {{ include "repositoryGenerator.image.quitQuit" . }} + imagePullPolicy: Always + command: + - /bin/sh + - "-c" + args: + - echo "waiting 10s for istio side cars to be up"; sleep 10s; + /app/ready.py --service-mesh-check {{ include "common.name" . }}-galera-db-migrator -t 45; + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- end }} restartPolicy: Never serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}} volumes: @@ -238,8 +289,6 @@ spec: app: {{ include "common.name" . }}-pg-config release: {{ include "common.release" . }} name: {{ include "common.name" . }}-pg-config - annotations: - sidecar.istio.io/inject: "false" spec: imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" @@ -270,6 +319,8 @@ spec: - /bin/sh - -cx - | + {{- if include "common.onServiceMesh" . }} + echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }} /dbcmd-config/db_migrator_pg_policy_init.sh env: - name: SQL_HOST @@ -287,6 +338,23 @@ spec: - name: PGPASSWORD {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "db-secret" "key" "password") | indent 10 }} resources: {{ include "common.resources" . | nindent 10 }} + {{- if (include "common.onServiceMesh" .) }} + - name: policy-service-mesh-wait-for-job-container + image: {{ include "repositoryGenerator.image.quitQuit" . }} + imagePullPolicy: Always + command: + - /bin/sh + - "-c" + args: + - echo "waiting 10s for istio side cars to be up"; sleep 10s; + /app/ready.py --service-mesh-check {{ include "common.name" . }}-pg-db-migrator -t 45; + env: + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- end }} restartPolicy: Never serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}} volumes: -- cgit 1.2.3-korg