[POLICY] Enhance liveness probes and metrics

- added logback prometheus support for xacml and drools.
- enabled enhanced healthchecks.
- enabled enhanced liveness probes for drools pdp to detect stuck PDP
application and restart it.
- add prometheus servicemonitor for drools.
- set latest docker image versions for drools-applications and xacml

Issue-ID: POLICY-3386
Issue-ID: POLICY-4089
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Change-Id: I0ffd9bf93452f8a7a63deb33f2103f9b80476d07
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>

7 files changed, 75 insertions, 10 deletions

diff --git a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf
index ff532ab5c1..57fa29bc3d 100755
--- a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf
+++ b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf
@@ -1,6 +1,6 @@
 {{/*
 # Copyright © 2017-2018 Amdocs, Bell Canada.
-# Modifications Copyright (C) 2018-2020 AT&T Intellectual Property.
+# Modifications Copyright (C) 2018-2020, 2022 AT&T Intellectual Property.
 # Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -43,6 +43,9 @@ REPOSITORY_OFFLINE={{.Values.nexus.offline}}
 SQL_HOST={{ .Values.db.name }}
 SQL_PORT=3306
 
+# Liveness
+LIVENESS_CONTROLLERS=*
+
 # AAF
 
 AAF={{.Values.aaf.enabled}}
diff --git a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml
index 2fc08e4e5d..a25a7e16d3 100755
--- a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml
+++ b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml
@@ -1,7 +1,7 @@
 <!--
   ============LICENSE_START=======================================================
    Copyright (C) 2020 Bell Canada. All rights reserved.
-   Modifications Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
+   Modifications Copyright (C) 2021-2022 AT&T Intellectual Property. All rights reserved.
   ================================================================================
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -150,6 +150,8 @@
         <appender-ref ref="AsyncStdOut" />
     </logger>
 
+    <appender name="PromLogback" class="io.prometheus.client.logback.InstrumentedAppender"/>
+
     <root level="INFO">
         <appender-ref ref="AsyncDebugOut" />
         <appender-ref ref="AsyncErrorOut" />
@@ -158,6 +160,7 @@
         <appender-ref ref="AsyncStdOut" />
         <appender-ref ref="AsyncMetricStdOut" />
         <appender-ref ref="AsyncTransactionStdOut" />
+        <appender-ref ref="PromLogback" />
     </root>
 
 </configuration>
diff --git a/kubernetes/policy/components/policy-drools-pdp/templates/serviceMonitor.yaml b/kubernetes/policy/components/policy-drools-pdp/templates/serviceMonitor.yaml
new file mode 100644
index 0000000000..1c04296e4f
--- /dev/null
+++ b/kubernetes/policy/components/policy-drools-pdp/templates/serviceMonitor.yaml
@@ -0,0 +1,23 @@
+{{/*
+# ============LICENSE_START=======================================================
+#  Copyright (c) 2022 AT&T Intellectual Property
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#  SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+*/}}
+
+{{- if .Values.prometheus.enabled }}
+{{ include "common.serviceMonitor" . }}
+{{- end }}
diff --git a/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml b/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml
index d389246b5c..7dee453771 100755
--- a/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml
+++ b/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml
@@ -1,6 +1,6 @@
 {{/*
 # Copyright © 2017 Amdocs, Bell Canada
-# Modifications Copyright © 2018-2020 AT&T Intellectual Property
+# Modifications Copyright © 2018-2020, 2022 AT&T Intellectual Property
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -83,10 +83,12 @@ spec:
           - containerPort: {{ .Values.service.externalPort2 }}
           {{- if eq .Values.liveness.enabled true }}
           livenessProbe:
-            tcpSocket:
-              port: {{ .Values.service.externalPort }}
+            httpGet:
+              path: /healthcheck/controllers
+              port: 6968
             initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
             periodSeconds: {{ .Values.liveness.periodSeconds }}
+            timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
           {{- end }}
           readinessProbe:
             tcpSocket:
diff --git a/kubernetes/policy/components/policy-drools-pdp/values.yaml b/kubernetes/policy/components/policy-drools-pdp/values.yaml
index 2ce7503015..344a600afe 100755
--- a/kubernetes/policy/components/policy-drools-pdp/values.yaml
+++ b/kubernetes/policy/components/policy-drools-pdp/values.yaml
@@ -1,6 +1,6 @@
 # Copyright © 2017 Amdocs
 # Copyright © 2017, 2021 Bell Canada
-# Modifications Copyright © 2018-2021 AT&T Intellectual Property
+# Modifications Copyright © 2018-2022 AT&T Intellectual Property
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,12 +30,18 @@ secrets:
     login: '{{ .Values.db.user }}'
     password: '{{ .Values.db.password }}'
     passwordPolicy: required
+  - uid: telemetry-creds
+    type: basicAuth
+    externalSecret: '{{ tpl (default "" .Values.telemetry.credsExternalSecret) . }}'
+    login: '{{ .Values.telemetry.user }}'
+    password: '{{ .Values.telemetry.password }}'
+    passwordPolicy: required
 
 #################################################################
 # Application configuration defaults.
 #################################################################
 # application image
-image: onap/policy-pdpd-cl:1.10.1
+image: onap/policy-pdpd-cl:1.10.2
 pullPolicy: Always
 
 # flag to enable debugging - application support required
@@ -51,7 +57,8 @@ affinity: {}
 # probe configuration parameters
 liveness:
   initialDelaySeconds: 180
-  periodSeconds: 10
+  periodSeconds: 60
+  timeoutSeconds: 10
   # necessary to disable liveness probe when setting breakpoints
   # in debugger so K8s doesn't restart unresponsive container
   enabled: true
@@ -195,3 +202,27 @@ serviceAccount:
   nameOverride: policy-drools-pdp
   roles:
     - read
+
+prometheus:
+  enabled: true
+
+metrics:
+  serviceMonitor:
+    # Override the labels based on the Prometheus config parameter: serviceMonitorSelector.
+    # The default operator for prometheus enforces the below label.
+    labels:
+      release: prometheus
+    enabled: true
+    port: policy-drools-pdp-9696
+    interval: 60s
+    isHttps: true
+    basicAuth:
+      enabled: true
+      externalSecretNameSuffix: policy-drools-pdp-telemetry-creds
+      externalSecretUserKey: login
+      externalSecretPasswordKey: password
+    selector:
+      app: '{{ include "common.name" . }}'
+      chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+      release: '{{ include "common.release" . }}'
+      heritage: '{{ .Release.Service }}'
diff --git a/kubernetes/policy/components/policy-xacml-pdp/resources/config/logback.xml b/kubernetes/policy/components/policy-xacml-pdp/resources/config/logback.xml
index cc7f8e56c1..fad7e72509 100755
--- a/kubernetes/policy/components/policy-xacml-pdp/resources/config/logback.xml
+++ b/kubernetes/policy/components/policy-xacml-pdp/resources/config/logback.xml
@@ -1,7 +1,7 @@
 <!--
   ============LICENSE_START=======================================================
    Copyright (C) 2020 Bell Canada. All rights reserved.
-   Modifications Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
+   Modifications Copyright (C) 2021-2022 AT&T Intellectual Property. All rights reserved.
   ================================================================================
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
@@ -95,10 +95,13 @@
         <appender-ref ref="AsyncStdOut" />
     </logger>
 
+    <appender name="PromLogback" class="io.prometheus.client.logback.InstrumentedAppender"/>
+
     <root level="INFO">
         <appender-ref ref="AsyncDebugOut" />
         <appender-ref ref="AsyncErrorOut" />
         <appender-ref ref="AsyncStdOut" />
+        <appender-ref ref="PromLogback" />
     </root>
 
 </configuration>
diff --git a/kubernetes/policy/components/policy-xacml-pdp/values.yaml b/kubernetes/policy/components/policy-xacml-pdp/values.yaml
index 504313832d..308b5a78af 100755
--- a/kubernetes/policy/components/policy-xacml-pdp/values.yaml
+++ b/kubernetes/policy/components/policy-xacml-pdp/values.yaml
@@ -83,7 +83,7 @@ certInitializer:
 # Application configuration defaults.
 #################################################################
 # application image
-image: onap/policy-xacml-pdp:2.6.1
+image: onap/policy-xacml-pdp:2.6.2
 pullPolicy: Always
 
 # flag to enable debugging - application support required