[POLICY] Correct clamp timeouts and enable sidecars in jobs again

Clamp pods take longer to start in "small" flavor case and require
60 seconds to startup.
Revert the Istio Sidecar removal in MariaDB jobs, as they are not
the root cause of the startup issue.

Issue-ID: OOM-3186

Signed-off-by: Andreas Geissler <andreas-geissler@telekom.de>
Change-Id: I0f3fd6a55e851640617bc3b0de8f96a0fe33d765

6 files changed, 86 insertions, 18 deletions

diff --git a/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml
index 3e1fdf4dff..20ea23c4ab 100755
--- a/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml
+++ b/kubernetes/policy/components/policy-clamp-ac-a1pms-ppnt/values.yaml
@@ -81,7 +81,7 @@ serviceMesh:
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
@@ -89,7 +89,7 @@ liveness:
   port: http-a1pms-api
 
 readiness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   port: http-a1pms-api
 
diff --git a/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml
index aeaf458daa..4842503fcc 100644
--- a/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml
+++ b/kubernetes/policy/components/policy-clamp-ac-http-ppnt/values.yaml
@@ -71,7 +71,7 @@ serviceMesh:
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
@@ -79,7 +79,7 @@ liveness:
   port: http-api
 
 readiness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   port: http-api
 
diff --git a/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml
index 0bbdf1307f..221d1405e1 100644
--- a/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml
+++ b/kubernetes/policy/components/policy-clamp-ac-k8s-ppnt/values.yaml
@@ -64,7 +64,7 @@ nodeSelector: {}
 affinity: {}
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
@@ -72,7 +72,7 @@ liveness:
   port: http-api
 
 readiness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   port: http-api
 
diff --git a/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml
index a793d615c2..237b946557 100755
--- a/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml
+++ b/kubernetes/policy/components/policy-clamp-ac-kserve-ppnt/values.yaml
@@ -71,7 +71,7 @@ serviceMesh:
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
@@ -79,7 +79,7 @@ liveness:
   port: kserve-api
 
 readiness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   port: kserve-api
 
diff --git a/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml b/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml
index a93fd866f9..c8387adc70 100644
--- a/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml
+++ b/kubernetes/policy/components/policy-clamp-ac-pf-ppnt/values.yaml
@@ -91,7 +91,7 @@ serviceMesh:
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
@@ -99,7 +99,7 @@ liveness:
   port: http-api
 
 readiness:
-  initialDelaySeconds: 20
+  initialDelaySeconds: 60
   periodSeconds: 10
   port: http-api
 
diff --git a/kubernetes/policy/templates/job.yaml b/kubernetes/policy/templates/job.yaml
index f0e91e8350..2503c6fd5f 100755
--- a/kubernetes/policy/templates/job.yaml
+++ b/kubernetes/policy/templates/job.yaml
@@ -32,8 +32,6 @@ spec:
         app: {{ include "common.name" . }}-galera-init
         release: {{ include "common.release" . }}
       name: {{ include "common.name" . }}-galera-init
-      annotations:
-        sidecar.istio.io/inject: "false"
     spec:
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
@@ -63,6 +61,8 @@ spec:
         - /bin/sh
         - -cx
         - |
+           {{- if include "common.onServiceMesh" . }}
+           echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }}
            /dbcmd-config/db.sh
         env:
         - name: MYSQL_ROOT_PASSWORD
@@ -74,6 +74,23 @@ spec:
         - name: MYSQL_PORT
           value: "{{ index .Values "mariadb-galera" "service" "internalPort" }}"
         resources: {{ include "common.resources" . | nindent 10 }}
+      {{- if (include "common.onServiceMesh" .) }}
+      - name: policy-service-mesh-wait-for-job-container
+        image: {{ include "repositoryGenerator.image.quitQuit" . }}
+        imagePullPolicy: Always
+        command:
+        - /bin/sh
+        - "-c"
+        args:
+        - echo "waiting 10s for istio side cars to be up"; sleep 10s;
+          /app/ready.py --service-mesh-check {{ include "common.name" . }}-galera-config -t 45;
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+      {{- end }}
       restartPolicy: Never
       serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}}
       volumes:
@@ -103,8 +120,6 @@ spec:
         app: {{ include "common.name" . }}-pg-init
         release: {{ include "common.release" . }}
       name: {{ include "common.name" . }}-pg-init
-      annotations:
-        sidecar.istio.io/inject: "false"
     spec:
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
@@ -121,6 +136,8 @@ spec:
           - /bin/sh
           - -cx
           - |
+             {{- if include "common.onServiceMesh" . }}
+             echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }}
              /docker-entrypoint-initdb.d/db-pg.sh
         env:
           - name: PG_ADMIN_PASSWORD
@@ -134,6 +151,23 @@ spec:
           - name: PG_PORT
             value: "{{ .Values.postgres.service.internalPort }}"
         resources: {{ include "common.resources" . | nindent 10 }}
+      {{- if (include "common.onServiceMesh" .) }}
+      - name: policy-service-mesh-wait-for-job-container
+        image: {{ include "repositoryGenerator.image.quitQuit" . }}
+        imagePullPolicy: Always
+        command:
+        - /bin/sh
+        - "-c"
+        args:
+        - echo "waiting 10s for istio side cars to be up"; sleep 10s;
+          /app/ready.py --service-mesh-check {{ include "common.name" . }}-pg-config -t 45;
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+      {{- end }}
       restartPolicy: Never
       serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}}
       volumes:
@@ -163,8 +197,6 @@ spec:
         app: {{ include "common.name" . }}-galera-config
         release: {{ include "common.release" . }}
       name: {{ include "common.name" . }}-galera-config
-      annotations:
-        sidecar.istio.io/inject: "false"
     spec:
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
@@ -195,6 +227,8 @@ spec:
         - /bin/sh
         - -cx
         - |
+           {{- if include "common.onServiceMesh" . }}
+           echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }}
            /dbcmd-config/db_migrator_policy_init.sh
         env:
         - name: SQL_HOST
@@ -210,6 +244,23 @@ spec:
         - name: SCRIPT_DIRECTORY
           value: "sql"
         resources: {{ include "common.resources" . | nindent 10 }}
+      {{- if (include "common.onServiceMesh" .) }}
+      - name: policy-service-mesh-wait-for-job-container
+        image: {{ include "repositoryGenerator.image.quitQuit" . }}
+        imagePullPolicy: Always
+        command:
+        - /bin/sh
+        - "-c"
+        args:
+        - echo "waiting 10s for istio side cars to be up"; sleep 10s;
+          /app/ready.py --service-mesh-check {{ include "common.name" . }}-galera-db-migrator -t 45;
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+      {{- end }}
       restartPolicy: Never
       serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}}
       volumes:
@@ -238,8 +289,6 @@ spec:
         app: {{ include "common.name" . }}-pg-config
         release: {{ include "common.release" . }}
       name: {{ include "common.name" . }}-pg-config
-      annotations:
-        sidecar.istio.io/inject: "false"
     spec:
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
@@ -270,6 +319,8 @@ spec:
           - /bin/sh
           - -cx
           - |
+             {{- if include "common.onServiceMesh" . }}
+             echo "waiting 15s for istio side cars to be up"; sleep 15s;{{- end }}
              /dbcmd-config/db_migrator_pg_policy_init.sh
         env:
         - name: SQL_HOST
@@ -287,6 +338,23 @@ spec:
         - name: PGPASSWORD
           {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "db-secret" "key" "password") | indent 10 }}
         resources: {{ include "common.resources" . | nindent 10 }}
+      {{- if (include "common.onServiceMesh" .) }}
+      - name: policy-service-mesh-wait-for-job-container
+        image: {{ include "repositoryGenerator.image.quitQuit" . }}
+        imagePullPolicy: Always
+        command:
+        - /bin/sh
+        - "-c"
+        args:
+        - echo "waiting 10s for istio side cars to be up"; sleep 10s;
+          /app/ready.py --service-mesh-check {{ include "common.name" . }}-pg-db-migrator -t 45;
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+      {{- end }}
       restartPolicy: Never
       serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}}
       volumes: