[AAI] Service monitor for scraping prometheus metrics

Changes are done for aai resources graphadmin and traversal
Session affinity for aai resources traversal and aai haproxy

Issue-ID: AAI-3599
Signed-off-by: leila <leila.nishimwe@bell.ca>
Change-Id: Ib3853c45051ca8e9c025bd0d6e700a08a54cb1a9

17 files changed, 209 insertions, 2 deletions

diff --git a/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties b/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties
index 27606021ef..6e64fd8400 100644
--- a/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties
+++ b/kubernetes/aai/components/aai-graphadmin/resources/config/application.properties
@@ -122,3 +122,8 @@ aperture.service.ssl.key-store-password=password(${KEYSTORE_JKS_PASSWORD})
 aperture.service.ssl.trust-store-password=password(${TRUSTSTORE_PASSWORD})
 {{ end }}
 aperture.service.timeout-in-milliseconds=300000
+
+#To Expose the Prometheus scraping endpoint
+management.port=8448
+endpoints.enabled=false
+management.security.enabled=false
\ No newline at end of file
diff --git a/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml b/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml
index 5241c6edf2..3e1479e335 100644
--- a/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml
+++ b/kubernetes/aai/components/aai-graphadmin/templates/deployment.yaml
@@ -143,6 +143,8 @@ spec:
           value: {{ .Values.service.internalPort | quote }}
         - name: INTERNAL_PORT_2
           value: {{ .Values.service.internalPort2 | quote }}
+        - name: INTERNAL_PORT_3
+          value: {{ .Values.service.internalPort3 | quote }}
         volumeMounts: {{ include "common.certInitializer.volumeMount" . | nindent 8 }}
         - mountPath: /etc/localtime
           name: localtime
@@ -173,6 +175,7 @@ spec:
         ports:
         - containerPort: {{ .Values.service.internalPort }}
         - containerPort: {{ .Values.service.internalPort2 }}
+        - containerPort: {{ .Values.service.internalPort3 }}
         lifecycle:
           # wait for active requests (long-running tasks) to be finished
           # Before the SIGTERM is invoked, Kubernetes exposes a preStop hook in the Pod.
diff --git a/kubernetes/aai/components/aai-graphadmin/templates/service.yaml b/kubernetes/aai/components/aai-graphadmin/templates/service.yaml
index cf4655361d..6350f858f1 100644
--- a/kubernetes/aai/components/aai-graphadmin/templates/service.yaml
+++ b/kubernetes/aai/components/aai-graphadmin/templates/service.yaml
@@ -40,11 +40,16 @@ spec:
   - port: {{ .Values.service.internalPort2 }}
     nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }}
     name: {{ .Values.service.portName2 }}
+  - port: {{ .Values.service.internalPort3 }}
+    nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort3 }}
+    name: {{ .Values.service.portName3 }}
   {{- else -}}
   - port: {{ .Values.service.internalPort }}
     name: {{ .Values.service.portName }}{{ (eq "true" (include "common.needTLS" .)) | ternary "s" "" }}
   - port: {{ .Values.service.internalPort2 }}
     name: {{ .Values.service.portName2 }}
+  - port: {{ .Values.service.internalPort3 }}
+    name: {{ .Values.service.portName3 }}
   {{- end}}
   selector:
     app: {{ include "common.name" . }}
diff --git a/kubernetes/aai/components/aai-graphadmin/templates/servicemonitor.yaml b/kubernetes/aai/components/aai-graphadmin/templates/servicemonitor.yaml
new file mode 100644
index 0000000000..c0d9f212b4
--- /dev/null
+++ b/kubernetes/aai/components/aai-graphadmin/templates/servicemonitor.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.metrics.serviceMonitor.enabled }}
+{{  include "common.serviceMonitor" . }}
+{{- end }}
\ No newline at end of file
diff --git a/kubernetes/aai/components/aai-graphadmin/values.yaml b/kubernetes/aai/components/aai-graphadmin/values.yaml
index 83da0bf91e..e8534e6a9f 100644
--- a/kubernetes/aai/components/aai-graphadmin/values.yaml
+++ b/kubernetes/aai/components/aai-graphadmin/values.yaml
@@ -240,6 +240,8 @@ service:
   internalPort: 8449
   portName2: tcp-5005
   internalPort2: 5005
+  portName3: aai-graphadmin-8448
+  internalPort3: 8448
   terminationGracePeriodSeconds: 120
 
 ingress:
@@ -284,6 +286,24 @@ resources:
       memory: 2Gi
   unlimited: {}
 
+metrics:
+  serviceMonitor:
+    enabled: false
+    targetPort: 8448
+    path: /prometheus
+    basicAuth:
+      enabled: false
+
+    selector:
+      app: '{{ include "common.name" . }}'
+      chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+      release: '{{ include "common.release" . }}'
+      heritage: '{{ .Release.Service }}'
+
+    relabelings: []
+
+    metricRelabelings: []
+
 # Not fully used for now
 securityContext:
   user_id: *user_id
diff --git a/kubernetes/aai/components/aai-resources/resources/config/application.properties b/kubernetes/aai/components/aai-resources/resources/config/application.properties
index 78a07f2a0f..3f33d71373 100644
--- a/kubernetes/aai/components/aai-resources/resources/config/application.properties
+++ b/kubernetes/aai/components/aai-resources/resources/config/application.properties
@@ -104,3 +104,20 @@ schema.service.ssl.trust-store={{ .Values.certInitializer.credsPath }}/{{ .Value
 schema.service.ssl.key-store-password=${KEYSTORE_PASSWORD}
 schema.service.ssl.trust-store-password=${TRUSTSTORE_PASSWORD}
 {{ end }}
+
+#to expose the Prometheus scraping endpoint
+management.port=8448
+management.endpoints.enabled-by-default=false
+management.security.enabled=false
+endpoints.enabled=false
+endpoints.info.enabled=false
+endpoints.prometheus.enabled=false
+endpoints.health.enabled=false
+management.metrics.web.server.auto-time-requests=false
+management.metrics.distribution.percentiles-histogram[http.server.requests]=true
+management.metrics.distribution.sla[http.server.requests]=20ms, 30ms, 40ms, 50ms, 60ms, 70ms, 80ms, 90ms, 100ms, 500ms, 1000ms, 5000ms, 7000ms
+#Add common tag for grouping all aai related metrics
+management.metrics.tags.group_id=aai
+#It is not advisable to use labels to store dimensions with high cardinality. Enable this option only for debug purposes. For more information: https://github.com/micrometer-metrics/micrometer/issues/1584
+scrape.uri.metrics=false
+
diff --git a/kubernetes/aai/components/aai-resources/templates/deployment.yaml b/kubernetes/aai/components/aai-resources/templates/deployment.yaml
index 021f0946ed..b5e89aa585 100644
--- a/kubernetes/aai/components/aai-resources/templates/deployment.yaml
+++ b/kubernetes/aai/components/aai-resources/templates/deployment.yaml
@@ -136,6 +136,8 @@ spec:
           value: {{ .Values.service.internalPort | quote }}
         - name: INTERNAL_PORT_2
           value: {{ .Values.service.internalPort2 | quote }}
+        - name: INTERNAL_PORT_3
+          value: {{ .Values.service.internalPort3 | quote }}
         volumeMounts: {{ include "common.certInitializer.volumeMount" . | nindent 8 }}
         - mountPath: /etc/localtime
           name: localtime
@@ -184,6 +186,7 @@ spec:
         ports:
         - containerPort: {{ .Values.service.internalPort }}
         - containerPort: {{ .Values.service.internalPort2 }}
+        - containerPort: {{ .Values.service.internalPort3 }}
         lifecycle:
           # wait for active requests (long-running tasks) to be finished
           # Before the SIGTERM is invoked, Kubernetes exposes a preStop hook in the Pod.
diff --git a/kubernetes/aai/components/aai-resources/templates/service.yaml b/kubernetes/aai/components/aai-resources/templates/service.yaml
index 46118461d1..8feb071bc6 100644
--- a/kubernetes/aai/components/aai-resources/templates/service.yaml
+++ b/kubernetes/aai/components/aai-resources/templates/service.yaml
@@ -34,12 +34,19 @@ spec:
   - port: {{ .Values.service.internalPort2 }}
     nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }}
     name: {{ .Values.service.portName2 }}
+  - port: {{ .Values.service.internalPort3 }}
+    nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort3 }}
+    name: {{ .Values.service.portName3 }}
   {{- else -}}
   - port: {{ .Values.service.internalPort }}
     name: {{ .Values.service.portName }}{{ (eq "true" (include "common.needTLS" .)) | ternary "s" "" }}
   - port: {{ .Values.service.internalPort2 }}
     name: {{ .Values.service.portName2 }}
+  - port: {{ .Values.service.internalPort3 }}
+    name: {{ .Values.service.portName3 }}
   {{- end }}
   selector:
     app: {{ include "common.name" . }}
     release: {{ include "common.release" . }}
+  clusterIP: None
+  sessionAffinity: {{ .Values.service.sessionAffinity }}
diff --git a/kubernetes/aai/components/aai-resources/templates/servicemonitor.yaml b/kubernetes/aai/components/aai-resources/templates/servicemonitor.yaml
new file mode 100644
index 0000000000..c0d9f212b4
--- /dev/null
+++ b/kubernetes/aai/components/aai-resources/templates/servicemonitor.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.metrics.serviceMonitor.enabled }}
+{{  include "common.serviceMonitor" . }}
+{{- end }}
\ No newline at end of file
diff --git a/kubernetes/aai/components/aai-resources/values.yaml b/kubernetes/aai/components/aai-resources/values.yaml
index 4952535ea5..df884428a7 100644
--- a/kubernetes/aai/components/aai-resources/values.yaml
+++ b/kubernetes/aai/components/aai-resources/values.yaml
@@ -231,7 +231,10 @@ service:
   internalPort: 8447
   portName2: tcp-5005
   internalPort2: 5005
+  portName3: aai-resources-8448
+  internalPort3: 8448
   terminationGracePeriodSeconds: 120
+  sessionAffinity: None
 
 ingress:
   enabled: false
@@ -270,6 +273,58 @@ resources:
       memory: 4Gi
   unlimited: {}
 
+metrics:
+  serviceMonitor:
+    enabled: false
+    targetPort: 8448
+    path: /prometheus
+    basicAuth:
+      enabled: false
+      externalSecretName: mysecretname
+      externalSecretUserKey: login
+      externalSecretPasswordKey: password
+
+    ## Namespace in which Prometheus is running
+    ##
+    # namespace: monitoring
+
+    ## Interval at which metrics should be scraped.
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    #interval: 30s
+
+    ## Timeout after which the scrape is ended
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    # scrapeTimeout: 10s
+
+    ## ServiceMonitor selector labels
+    ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration
+    ##
+    selector:
+      app: '{{ include "common.name" . }}'
+      chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+      release: '{{ include "common.release" . }}'
+      heritage: '{{ .Release.Service }}'
+
+    ## RelabelConfigs to apply to samples before scraping
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
+    ## Value is evalued as a template
+    ##
+    relabelings: []
+
+    ## MetricRelabelConfigs to apply to samples before ingestion
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
+    ## Value is evalued as a template
+    ##
+    metricRelabelings: []
+    #  - sourceLabels:
+    #      - "__name__"
+    #    targetLabel: "__name__"
+    #    action: replace
+    #    regex: '(.*)'
+    #    replacement: 'example_prefix_$1'
+
 #Pods Service Account
 serviceAccount:
   nameOverride: aai-resources
diff --git a/kubernetes/aai/components/aai-traversal/resources/config/application.properties b/kubernetes/aai/components/aai-traversal/resources/config/application.properties
index 24a5241fe2..3022b17f97 100644
--- a/kubernetes/aai/components/aai-traversal/resources/config/application.properties
+++ b/kubernetes/aai/components/aai-traversal/resources/config/application.properties
@@ -106,3 +106,19 @@ schema.service.ssl.trust-store={{ .Values.certInitializer.credsPath }}/{{ .Value
 schema.service.ssl.key-store-password=${KEYSTORE_PASSWORD}
 schema.service.ssl.trust-store-password=${TRUSTSTORE_PASSWORD}
 {{ end }}
+
+#to expose the Prometheus scraping endpoint
+management.port=8448
+management.endpoints.enabled-by-default=false
+management.security.enabled=false
+endpoints.enabled=false
+endpoints.info.enabled=false
+endpoints.prometheus.enabled=false
+endpoints.health.enabled=false
+management.metrics.web.server.auto-time-requests=false
+management.metrics.distribution.percentiles-histogram[http.server.requests]=true
+management.metrics.distribution.sla[http.server.requests]=20ms, 30ms, 40ms, 50ms, 60ms, 70ms, 80ms, 90ms, 100ms, 500ms, 1000ms, 5000ms, 7000ms
+#Add common tag for grouping all aai related metrics
+management.metrics.tags.group_id=aai
+#It is not advisable to use labels to store dimensions with high cardinality. Enable this option only for debug purposes. For more information: https://github.com/micrometer-metrics/micrometer/issues/1584
+scrape.uri.metrics=false
\ No newline at end of file
diff --git a/kubernetes/aai/components/aai-traversal/templates/deployment.yaml b/kubernetes/aai/components/aai-traversal/templates/deployment.yaml
index ed8cde7a88..9300c53a9c 100644
--- a/kubernetes/aai/components/aai-traversal/templates/deployment.yaml
+++ b/kubernetes/aai/components/aai-traversal/templates/deployment.yaml
@@ -155,6 +155,8 @@ spec:
           value: {{ .Values.service.internalPort | quote }}
         - name: INTERNAL_PORT_2
           value: {{ .Values.service.internalPort2 | quote }}
+        - name: INTERNAL_PORT_3
+          value: {{ .Values.service.internalPort3 | quote }}
         volumeMounts: {{ include "common.certInitializer.volumeMount" . | nindent 8 }}
         - mountPath: /etc/localtime
           name: localtime
@@ -205,6 +207,7 @@ spec:
         ports:
         - containerPort: {{ .Values.service.internalPort }}
         - containerPort: {{ .Values.service.internalPort2 }}
+        - containerPort: {{ .Values.service.internalPort3 }}
         lifecycle:
           # wait for active requests (long-running tasks) to be finished
           # Before the SIGTERM is invoked, Kubernetes exposes a preStop hook in the Pod.
diff --git a/kubernetes/aai/components/aai-traversal/templates/service.yaml b/kubernetes/aai/components/aai-traversal/templates/service.yaml
index b54b6be644..daf1758890 100644
--- a/kubernetes/aai/components/aai-traversal/templates/service.yaml
+++ b/kubernetes/aai/components/aai-traversal/templates/service.yaml
@@ -34,12 +34,19 @@ spec:
   - port: {{ .Values.service.internalPort2 }}
     nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }}
     name: {{ .Values.service.portName2 }}
+  - port: {{ .Values.service.internalPort3 }}
+    nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort3 }}
+    name: {{ .Values.service.portName3 }}
   {{- else -}}
   - port: {{ .Values.service.internalPort }}
     name: {{ .Values.service.portName }}{{ (eq "true" (include "common.needTLS" .)) | ternary "s" "" }}
   - port: {{ .Values.service.internalPort2 }}
     name: {{ .Values.service.portName2 }}
+  - port: {{ .Values.service.internalPort3 }}
+    name: {{ .Values.service.portName3 }}
   {{- end }}
   selector:
     app: {{ include "common.name" . }}
     release: {{ include "common.release" . }}
+  clusterIP: None
+  sessionAffinity: {{ .Values.service.sessionAffinity }}
diff --git a/kubernetes/aai/components/aai-traversal/templates/servicemonitor.yaml b/kubernetes/aai/components/aai-traversal/templates/servicemonitor.yaml
new file mode 100644
index 0000000000..c0d9f212b4
--- /dev/null
+++ b/kubernetes/aai/components/aai-traversal/templates/servicemonitor.yaml
@@ -0,0 +1,3 @@
+{{- if .Values.metrics.serviceMonitor.enabled }}
+{{  include "common.serviceMonitor" . }}
+{{- end }}
\ No newline at end of file
diff --git a/kubernetes/aai/components/aai-traversal/values.yaml b/kubernetes/aai/components/aai-traversal/values.yaml
index b80da08566..f6ed17a16b 100644
--- a/kubernetes/aai/components/aai-traversal/values.yaml
+++ b/kubernetes/aai/components/aai-traversal/values.yaml
@@ -259,7 +259,10 @@ service:
   internalPort: 8446
   portName2: tcp-5005
   internalPort2: 5005
+  portName3: aai-traversal-8448
+  internalPort3: 8448
   terminationGracePeriodSeconds: 120
+  sessionAffinity: None
 
 ingress:
   enabled: false
@@ -283,6 +286,58 @@ resources:
       memory: 4Gi
   unlimited: {}
 
+metrics:
+  serviceMonitor:
+    enabled: false
+    targetPort: 8448
+    path: /prometheus
+    basicAuth:
+      enabled: false
+      externalSecretName: mysecretname
+      externalSecretUserKey: login
+      externalSecretPasswordKey: password
+
+    ## Namespace in which Prometheus is running
+    ##
+    # namespace: monitoring
+
+    ## Interval at which metrics should be scraped.
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    #interval: 30s
+
+    ## Timeout after which the scrape is ended
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    # scrapeTimeout: 10s
+
+    ## ServiceMonitor selector labels
+    ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration
+    ##
+    selector:
+      app: '{{ include "common.name" . }}'
+      chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+      release: '{{ include "common.release" . }}'
+      heritage: '{{ .Release.Service }}'
+
+    ## RelabelConfigs to apply to samples before scraping
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
+    ## Value is evalued as a template
+    ##
+    relabelings: []
+
+    ## MetricRelabelConfigs to apply to samples before ingestion
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#relabelconfig
+    ## Value is evalued as a template
+    ##
+    metricRelabelings: []
+    #  - sourceLabels:
+    #      - "__name__"
+    #    targetLabel: "__name__"
+    #    action: replace
+    #    regex: '(.*)'
+    #    replacement: 'example_prefix_$1'
+
 #Pods Service Account
 serviceAccount:
   nameOverride: aai-traversal
diff --git a/kubernetes/aai/templates/service.yaml b/kubernetes/aai/templates/service.yaml
index c5fecf6a74..a8c3c3957e 100644
--- a/kubernetes/aai/templates/service.yaml
+++ b/kubernetes/aai/templates/service.yaml
@@ -36,6 +36,7 @@ spec:
   selector:
     app: {{ include "common.name" . }}
   clusterIP: {{ .Values.service.aaiServiceClusterIp }}
+  sessionAffinity: {{ .Values.service.sessionAffinity }}
 ---
 apiVersion: v1
 kind: Service
@@ -74,5 +75,4 @@ spec:
   type: {{ .Values.metricsService.type }}
   selector:
     app: {{ include "common.name" . }}
-  clusterIP: None
-
+  clusterIP: None
\ No newline at end of file
diff --git a/kubernetes/aai/values.yaml b/kubernetes/aai/values.yaml
index fedd64078f..c19f193736 100644
--- a/kubernetes/aai/values.yaml
+++ b/kubernetes/aai/values.yaml
@@ -407,6 +407,8 @@ service:
   externalPlainPort: 80
   internalPlainPort: 8080
   nodeport: 33
+  aaiServiceClusterIp:
+  sessionAffinity: None
 
 metricsService:
   type: ClusterIP