[SDC] Use Startup probes

Instead of long initial delay on readiness and liveness probes, use
startup probes and be more aggressive on readiness and liveness.

Issue-ID: OOM-2743
Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com>
Change-Id: Ibde8d10dad150fcad5740533d0bdd0926a44ccc1

12 files changed, 161 insertions, 15 deletions

diff --git a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml
index 44439869cc..28e9c1369c 100644
--- a/kubernetes/sdc/components/sdc-be/templates/deployment.yaml
+++ b/kubernetes/sdc/components/sdc-be/templates/deployment.yaml
@@ -124,6 +124,8 @@ spec:
             initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
             periodSeconds: {{ .Values.liveness.periodSeconds }}
             timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
+            successThreshold: {{ .Values.liveness.successThreshold }}
+            failureThreshold: {{ .Values.liveness.failureThreshold }}
           {{ end }}
           readinessProbe:
             exec:
@@ -132,6 +134,18 @@ spec:
             initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
             periodSeconds: {{ .Values.readiness.periodSeconds }}
             timeoutSeconds: {{ .Values.readiness.timeoutSeconds }}
+            successThreshold: {{ .Values.readiness.successThreshold }}
+            failureThreshold: {{ .Values.readiness.failureThreshold }}
+          resources: {{ include "common.resources" . | nindent 12 }}
+          startupProbe:
+            exec:
+              command:
+              - "/var/lib/jetty/ready-probe.sh"
+            initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startup.periodSeconds }}
+            timeoutSeconds: {{ .Values.startup.timeoutSeconds }}
+            successThreshold: {{ .Values.startup.successThreshold }}
+            failureThreshold: {{ .Values.startup.failureThreshold }}
           resources: {{ include "common.resources" . | nindent 12 }}
           env:
           - name: ENVNAME
diff --git a/kubernetes/sdc/components/sdc-be/values.yaml b/kubernetes/sdc/components/sdc-be/values.yaml
index 045136aef0..1277fc9176 100644
--- a/kubernetes/sdc/components/sdc-be/values.yaml
+++ b/kubernetes/sdc/components/sdc-be/values.yaml
@@ -83,18 +83,29 @@ affinity: {}
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 120
+  initialDelaySeconds: 1
   periodSeconds: 10
   timeoutSeconds: 5
+  successThreshold: 1
+  failureThreshold: 3
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
   port: api
   enabled: true
 
 readiness:
-  initialDelaySeconds: 60
+  initialDelaySeconds: 1
   periodSeconds: 10
   timeoutSeconds: 5
+  successThreshold: 1
+  failureThreshold: 3
+
+startup:
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  timeoutSeconds: 5
+  successThreshold: 1
+  failureThreshold: 60
 
 service:
   type: NodePort
diff --git a/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml b/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml
index 45c7bc85b6..0a5c0a3445 100644
--- a/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml
+++ b/kubernetes/sdc/components/sdc-fe/templates/deployment.yaml
@@ -117,13 +117,25 @@ spec:
             initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
             periodSeconds: {{ .Values.liveness.periodSeconds }}
             timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
+            successThreshold: {{ .Values.liveness.successThreshold }}
+            failureThreshold: {{ .Values.liveness.failureThreshold }}
           {{ end }}
           readinessProbe:
             tcpSocket:
               port: {{ .Values.service.internalPort2 }}
             initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
             periodSeconds: {{ .Values.readiness.periodSeconds }}
-            timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
+            timeoutSeconds: {{ .Values.readiness.timeoutSeconds }}
+            successThreshold: {{ .Values.readiness.successThreshold }}
+            failureThreshold: {{ .Values.readiness.failureThreshold }}
+          startupProbe:
+            tcpSocket:
+              port: {{ .Values.service.internalPort2 }}
+            initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startup.periodSeconds }}
+            timeoutSeconds: {{ .Values.startup.timeoutSeconds }}
+            successThreshold: {{ .Values.startup.successThreshold }}
+            failureThreshold: {{ .Values.startup.failureThreshold }}
           resources: {{ include "common.resources" . | nindent 12 }}
           env:
           - name: ENVNAME
diff --git a/kubernetes/sdc/components/sdc-fe/values.yaml b/kubernetes/sdc/components/sdc-fe/values.yaml
index 7aad4c13cb..dcebc475ad 100644
--- a/kubernetes/sdc/components/sdc-fe/values.yaml
+++ b/kubernetes/sdc/components/sdc-fe/values.yaml
@@ -76,17 +76,28 @@ affinity: {}
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 10
-  periodSeconds: 60
+  initialDelaySeconds: 1
+  periodSeconds: 10
   timeoutSeconds: 15
+  successThreshold: 1
+  failureThreshold: 3
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
   enabled: true
 
 readiness:
+  initialDelaySeconds: 1
+  periodSeconds: 10
+  timeoutSeconds: 15
+  successThreshold: 1
+  failureThreshold: 3
+
+startup:
   initialDelaySeconds: 10
-  periodSeconds: 60
+  periodSeconds: 10
   timeoutSeconds: 15
+  successThreshold: 1
+  failureThreshold: 60
 
 service:
   #Example service definition with external, internal and node ports.
diff --git a/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml b/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml
index 08228ad99c..f736a174db 100644
--- a/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml
+++ b/kubernetes/sdc/components/sdc-helm-validator/templates/deployment.yaml
@@ -40,5 +40,15 @@ spec:
               port: {{ .Values.liveness.port }}
             initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
             periodSeconds: {{ .Values.liveness.periodSeconds }}
+            successThreshold: {{ .Values.liveness.successThreshold }}
+            failureThreshold: {{ .Values.liveness.failureThreshold }}
+          startupProbe:
+            httpGet:
+              path: {{ .Values.startup.path }}
+              port: {{ .Values.startup.port }}
+            initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startup.periodSeconds }}
+            successThreshold: {{ .Values.startup.successThreshold }}
+            failureThreshold: {{ .Values.startup.failureThreshold }}
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
diff --git a/kubernetes/sdc/components/sdc-helm-validator/values.yaml b/kubernetes/sdc/components/sdc-helm-validator/values.yaml
index 9c0d90649f..ede80a6af0 100644
--- a/kubernetes/sdc/components/sdc-helm-validator/values.yaml
+++ b/kubernetes/sdc/components/sdc-helm-validator/values.yaml
@@ -31,14 +31,24 @@ service:
       port: *svc_port
 
 liveness:
-  initialDelaySeconds: 30
-  periodSeconds: 30
+  initialDelaySeconds: 1
+  periodSeconds: 10
   path: /actuator/health
+  successThreshold: 1
+  failureThreshold: 3
   port: *port
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
   enabled: true
 
+startup:
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  path: /actuator/health
+  successThreshold: 1
+  failureThreshold: 12
+  port: *port
+
 flavor: small
 resources:
   small:
diff --git a/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml
index 5c530fea72..eb85465855 100644
--- a/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml
+++ b/kubernetes/sdc/components/sdc-onboarding-be/templates/deployment.yaml
@@ -128,6 +128,8 @@ spec:
             initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
             periodSeconds: {{ .Values.liveness.periodSeconds }}
             timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
+            successThreshold: {{ .Values.liveness.successThreshold }}
+            failureThreshold: {{ .Values.liveness.failureThreshold }}
           {{ end }}
           readinessProbe:
             exec:
@@ -135,7 +137,18 @@ spec:
               - "/var/lib/jetty/ready-probe.sh"
             initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
             periodSeconds: {{ .Values.readiness.periodSeconds }}
-            timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
+            timeoutSeconds: {{ .Values.readiness.timeoutSeconds }}
+            successThreshold: {{ .Values.readiness.successThreshold }}
+            failureThreshold: {{ .Values.readiness.failureThreshold }}
+          startupProbe:
+            exec:
+              command:
+              - "/var/lib/jetty/ready-probe.sh"
+            initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startup.periodSeconds }}
+            timeoutSeconds: {{ .Values.startup.timeoutSeconds }}
+            successThreshold: {{ .Values.startup.successThreshold }}
+            failureThreshold: {{ .Values.startup.failureThreshold }}
           resources: {{ include "common.resources" . | nindent 12 }}
           env:
           - name: ENVNAME
diff --git a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml
index 7e4dddd124..ddff1d3f1a 100644
--- a/kubernetes/sdc/components/sdc-onboarding-be/values.yaml
+++ b/kubernetes/sdc/components/sdc-onboarding-be/values.yaml
@@ -83,17 +83,28 @@ affinity: {}
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 120
-  periodSeconds: 60
+  initialDelaySeconds: 1
+  periodSeconds: 10
   timeoutSeconds: 15
+  successThreshold: 1
+  failureThreshold: 3
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
   enabled: true
 
 readiness:
-  initialDelaySeconds: 120
-  periodSeconds: 60
+  initialDelaySeconds: 1
+  periodSeconds: 10
   timeoutSeconds: 15
+  successThreshold: 1
+  failureThreshold: 3
+
+startup:
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  timeoutSeconds: 15
+  successThreshold: 1
+  failureThreshold: 60
 
 service:
   type: ClusterIP
diff --git a/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml b/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml
index 9defb8e1ce..de75092f8a 100644
--- a/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml
+++ b/kubernetes/sdc/components/sdc-wfd-be/templates/deployment.yaml
@@ -88,12 +88,23 @@ spec:
               port: {{ template "wfd-be.internalPort" . }}
             initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
             periodSeconds: {{ .Values.liveness.periodSeconds }}
+            successThreshold: {{ .Values.liveness.successThreshold }}
+            failureThreshold: {{ .Values.liveness.failureThreshold }}
           {{ end }}
           readinessProbe:
             tcpSocket:
               port: {{ template "wfd-be.internalPort" . }}
             initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
             periodSeconds: {{ .Values.readiness.periodSeconds }}
+            successThreshold: {{ .Values.readiness.successThreshold }}
+            failureThreshold: {{ .Values.readiness.failureThreshold }}
+          startupProbe:
+            tcpSocket:
+              port: {{ template "wfd-be.internalPort" . }}
+            initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startup.periodSeconds }}
+            successThreshold: {{ .Values.startup.successThreshold }}
+            failureThreshold: {{ .Values.startup.failureThreshold }}
           env:
           - name: JAVA_OPTIONS
             value: {{ .Values.config.javaOptions }}
diff --git a/kubernetes/sdc/components/sdc-wfd-be/values.yaml b/kubernetes/sdc/components/sdc-wfd-be/values.yaml
index 4aebe7ab9a..0d75ae3c8b 100644
--- a/kubernetes/sdc/components/sdc-wfd-be/values.yaml
+++ b/kubernetes/sdc/components/sdc-wfd-be/values.yaml
@@ -101,6 +101,28 @@ readiness:
   initialDelaySeconds: 60
   periodSeconds: 10
 
+# probe configuration parameters
+liveness:
+  initialDelaySeconds: 1
+  periodSeconds: 10
+  successThreshold: 1
+  failureThreshold: 3
+  # necessary to disable liveness probe when setting breakpoints
+  # in debugger so K8s doesn't restart unresponsive container
+  enabled: true
+
+readiness:
+  initialDelaySeconds: 1
+  periodSeconds: 10
+  successThreshold: 1
+  failureThreshold: 3
+
+startup:
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  successThreshold: 1
+  failureThreshold: 60
+
 service:
   type: NodePort
   portName: sdc-wfd-be
diff --git a/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml b/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml
index 7a8cf8fb34..b8073d723d 100644
--- a/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml
+++ b/kubernetes/sdc/components/sdc-wfd-fe/templates/deployment.yaml
@@ -105,12 +105,23 @@ spec:
               port: {{ template "wfd-fe.internalPort" . }}
             initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
             periodSeconds: {{ .Values.liveness.periodSeconds }}
+            successThreshold: {{ .Values.liveness.successThreshold }}
+            failureThreshold: {{ .Values.liveness.failureThreshold }}
           {{ end }}
           readinessProbe:
             tcpSocket:
               port: {{ template "wfd-fe.internalPort" . }}
             initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
             periodSeconds: {{ .Values.readiness.periodSeconds }}
+            successThreshold: {{ .Values.readiness.successThreshold }}
+            failureThreshold: {{ .Values.readiness.failureThreshold }}
+          startupProbe:
+            tcpSocket:
+              port: {{ template "wfd-fe.internalPort" . }}
+            initialDelaySeconds: {{ .Values.startup.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startup.periodSeconds }}
+            successThreshold: {{ .Values.startup.successThreshold }}
+            failureThreshold: {{ .Values.startup.failureThreshold }}
           env:
           - name: ENVNAME
             value: {{ .Values.env.name }}
diff --git a/kubernetes/sdc/components/sdc-wfd-fe/values.yaml b/kubernetes/sdc/components/sdc-wfd-fe/values.yaml
index ff8aebf6b2..235931bd7d 100644
--- a/kubernetes/sdc/components/sdc-wfd-fe/values.yaml
+++ b/kubernetes/sdc/components/sdc-wfd-fe/values.yaml
@@ -77,15 +77,25 @@ affinity: {}
 
 # probe configuration parameters
 liveness:
-  initialDelaySeconds: 60
+  initialDelaySeconds: 1
   periodSeconds: 10
+  successThreshold: 1
+  failureThreshold: 3
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
   enabled: true
 
 readiness:
-  initialDelaySeconds: 60
+  initialDelaySeconds: 1
   periodSeconds: 10
+  successThreshold: 1
+  failureThreshold: 3
+
+startup:
+  initialDelaySeconds: 10
+  periodSeconds: 10
+  successThreshold: 1
+  failureThreshold: 60
 
 service:
   type: NodePort