From 7000a7c8ad6acdbfbaed54c81b030f9b7ad190b1 Mon Sep 17 00:00:00 2001 From: Maciej Wereski Date: Thu, 16 Dec 2021 12:24:06 +0100 Subject: [DCAEGEN2] Use common.log template Until now DCAEGEN2 used its own templates instead of common ones. This change switches filebeat sidecar generation to common.log templates. Other ONAP components already use it. Filebeat is depreceated and with use of common templates it is possible to disable it globally. This patch also changes ConfigMap usage from per component ConfigMap to one ConfigMap for all of DCAEGEN2 components. This is done to simplify generated file as there are no differences between per component ConfigMaps. Issue-ID: OOM-1 Signed-off-by: Maciej Wereski Change-Id: I87ffe4890b9a4d79edfe0669750dbf21b4a7b340 --- .../templates/_configmap.tpl | 15 +--- .../templates/_deployment.tpl | 32 ++------ .../templates/_filebeat-config.tpl | 86 ---------------------- .../dcae-bbs-eventprocessor-ms/values.yaml | 8 +- .../components/dcae-datafile-collector/values.yaml | 8 +- .../components/dcae-datalake-admin-ui/values.yaml | 8 +- .../components/dcae-datalake-des/values.yaml | 10 ++- .../components/dcae-datalake-feeder/values.yaml | 8 +- .../components/dcae-heartbeat/values.yaml | 8 +- .../components/dcae-hv-ves-collector/values.yaml | 8 +- .../components/dcae-kpi-ms/values.yaml | 8 +- .../components/dcae-pm-mapper/values.yaml | 8 +- .../components/dcae-pmsh/values.yaml | 8 +- .../components/dcae-prh/values.yaml | 8 +- .../components/dcae-restconf-collector/values.yaml | 8 +- .../components/dcae-slice-analysis-ms/values.yaml | 8 +- .../components/dcae-snmptrap-collector/values.yaml | 8 +- .../components/dcae-son-handler/values.yaml | 8 +- .../components/dcae-tcagen2/values.yaml | 8 +- .../components/dcae-ves-collector/values.yaml | 8 +- .../components/dcae-ves-mapper/values.yaml | 8 +- .../resources/config/log/filebeat/filebeat.yml | 84 +++++++++++++++++++++ .../dcaegen2-services/templates/configmap.yaml | 4 +- kubernetes/dcaegen2-services/values.yaml | 30 ++++++++ 24 files changed, 232 insertions(+), 165 deletions(-) delete mode 100644 kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_filebeat-config.tpl create mode 100644 kubernetes/dcaegen2-services/resources/config/log/filebeat/filebeat.yml (limited to 'kubernetes') diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl index 5313b0782a..f76be4c190 100644 --- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl +++ b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_configmap.tpl @@ -52,19 +52,6 @@ data: application_config.yaml: | {{ $appConf | toYaml | indent 4 }} -{{- if .Values.logDirectory }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "common.fullname" . }}-filebeat-configmap - namespace: {{ include "common.namespace" . }} - labels: {{ include "common.labels" . | nindent 6 }} -data: - filebeat.yml: |- -{{ include "dcaegen2-services-common.filebeatConfiguration" . | indent 4 }} -{{- end }} - {{- if .Values.drFeedConfig }} --- apiVersion: v1 @@ -124,4 +111,4 @@ data: {{ $topics | toJson | indent 2 }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl index 5ba7d2977a..ef49f8c5d4 100644 --- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl +++ b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_deployment.tpl @@ -222,7 +222,7 @@ policies: */}} {{- define "dcaegen2-services-common.microserviceDeployment" -}} -{{- $logDir := default "" .Values.logDirectory -}} +{{- $logDir := default "" .Values.log.path -}} {{- $certDir := default "" .Values.certDirectory . -}} {{- $tlsServer := default "" .Values.tlsServer -}} {{- $commonRelease := print (include "common.release" .) -}} @@ -355,7 +355,7 @@ spec: name: app-config-input {{- if $logDir }} - mountPath: {{ $logDir}} - name: component-log + name: logs {{- end }} {{- if $certDir }} - mountPath: {{ $certDir }} @@ -370,24 +370,7 @@ spec: {{- end }} {{- include "dcaegen2-services-common._externalVolumeMounts" . | nindent 8 }} {{- if $logDir }} - - image: {{ include "repositoryGenerator.image.logging" . }} - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - name: filebeat - env: - - name: POD_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - resources: {{ include "common.resources" . | nindent 2 }} - volumeMounts: - - mountPath: /var/log/onap/{{ include "common.name" . }} - name: component-log - - mountPath: /usr/share/filebeat/data - name: filebeat-data - - mountPath: /usr/share/filebeat/filebeat.yml - name: filebeat-conf - subPath: filebeat.yml + {{ include "common.log.sidecar" . | nindent 6 }} {{- end }} {{- if $policy }} - image: {{ include "repositoryGenerator.repository" . }}/{{ .Values.dcaePolicySyncImage }} @@ -448,13 +431,8 @@ spec: name: app-config {{- if $logDir }} - emptyDir: {} - name: component-log - - emptyDir: {} - name: filebeat-data - - configMap: - defaultMode: 420 - name: {{ include "common.fullname" . }}-filebeat-configmap - name: filebeat-conf + name: logs + {{ include "common.log.volumes" (dict "dot" . "configMapNamePrefix" (tpl .Values.logConfigMapNamePrefix . )) | nindent 6 }} {{- end }} {{- if $certDir }} - emptyDir: {} diff --git a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_filebeat-config.tpl b/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_filebeat-config.tpl deleted file mode 100644 index a402517068..0000000000 --- a/kubernetes/dcaegen2-services/common/dcaegen2-services-common/templates/_filebeat-config.tpl +++ /dev/null @@ -1,86 +0,0 @@ -{{/* -#============LICENSE_START======================================================== -# ================================================================================ -# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. -# Modifications Copyright © 2018 Amdocs, Bell Canada -# Copyright (c) 2021 J. F. Lucas. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============LICENSE_END========================================================= -*/}} -{{/* -dcaegen2-services-common.filebeatConfiguration: -This template generates configuration data for filebeat (log file aggregation). - -The template is used to create a configMap mounted by a filebeat sidecar pod -running alongside a DCAE microservice pod. - -See dcaegen2-services-common.configMap for more information. -*/}} -{{- define "dcaegen2-services-common.filebeatConfiguration" -}} -filebeat.prospectors: -#it is mandatory, in our case it's log -- input_type: log - #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. - paths: - - /var/log/onap/*/*/*/*.log - - /var/log/onap/*/*/*.log - - /var/log/onap/*/*.log - #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive - ignore_older: 48h - # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit - clean_inactive: 96h - - -# Name of the registry file. If a relative path is used, it is considered relative to the -# data path. Else full qualified file name. -#filebeat.registry_file: ${path.data}/registry - - -output.logstash: - #List of logstash server ip addresses with port number. - #But, in our case, this will be the loadbalancer IP address. - #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. - hosts: ["{{.Values.filebeatConfig.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.filebeatConfig.logstashPort}}"] - #If enable will do load balancing among availabe Logstash, automatically. - loadbalance: true - - #The list of root certificates for server verifications. - #If certificate_authorities is empty or not set, the trusted - #certificate authorities of the host system are used. - #ssl.certificate_authorities: $ssl.certificate_authorities - - #The path to the certificate for SSL client authentication. If the certificate is not specified, - #client authentication is not available. - #ssl.certificate: $ssl.certificate - - #The client certificate key used for client authentication. - #ssl.key: $ssl.key - - #The passphrase used to decrypt an encrypted key stored in the configured key file - #ssl.key_passphrase: $ssl.key_passphrase - -logging: - level: debug - - # enable file rotation with default configuration - to_files: true - - # do not log to syslog - to_syslog: false - - files: - path: /usr/share/filebeat/logs - name: mybeat.log - keepfiles: 7 -{{- end -}} \ No newline at end of file diff --git a/kubernetes/dcaegen2-services/components/dcae-bbs-eventprocessor-ms/values.yaml b/kubernetes/dcaegen2-services/components/dcae-bbs-eventprocessor-ms/values.yaml index 572e812cf3..f55aeecdbd 100644 --- a/kubernetes/dcaegen2-services/components/dcae-bbs-eventprocessor-ms/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-bbs-eventprocessor-ms/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -59,8 +60,11 @@ image: onap/org.onap.dcaegen2.services.components.bbs-event-processor:2.1.1 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /opt/app/bbs-event-processor/logs +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /opt/app/bbs-event-processor/logs +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml index 2342470877..9bdd0b7887 100644 --- a/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-datafile-collector/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -44,8 +45,11 @@ image: onap/org.onap.dcaegen2.collectors.datafile.datafile-app-server:1.6.1 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-datalake-admin-ui/values.yaml b/kubernetes/dcaegen2-services/components/dcae-datalake-admin-ui/values.yaml index 0553b52265..22a5a5b1cb 100644 --- a/kubernetes/dcaegen2-services/components/dcae-datalake-admin-ui/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-datalake-admin-ui/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -52,8 +53,11 @@ consulLoaderImage: onap/org.onap.dcaegen2.deployments.consul-loader-container:1. image: onap/org.onap.dcaegen2.services.datalakeadminui:1.1.1 # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/datalake-admin-ui +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/datalake-admin-ui +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-datalake-des/values.yaml b/kubernetes/dcaegen2-services/components/dcae-datalake-des/values.yaml index 9373e8256a..051a7a4673 100644 --- a/kubernetes/dcaegen2-services/components/dcae-datalake-des/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-datalake-des/values.yaml @@ -21,10 +21,11 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. -#B +# ################################################################# filebeatConfig: logstashServiceName: log-ls @@ -58,8 +59,11 @@ consulLoaderImage: onap/org.onap.dcaegen2.deployments.consul-loader-container:1. image: onap/org.onap.dcaegen2.services.datalake.exposure.service:1.1.1 # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/datalake +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/datalake +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-datalake-feeder/values.yaml b/kubernetes/dcaegen2-services/components/dcae-datalake-feeder/values.yaml index 2452dc8a18..07306e1286 100644 --- a/kubernetes/dcaegen2-services/components/dcae-datalake-feeder/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-datalake-feeder/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -60,8 +61,11 @@ image: onap/org.onap.dcaegen2.services.datalakefeeder:1.1.1 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/datalake +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/datalake +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-heartbeat/values.yaml b/kubernetes/dcaegen2-services/components/dcae-heartbeat/values.yaml index bbf815d658..4ed0a83677 100644 --- a/kubernetes/dcaegen2-services/components/dcae-heartbeat/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-heartbeat/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -60,8 +61,11 @@ image: onap/org.onap.dcaegen2.services.heartbeat:2.3.1 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/heartbeat +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/heartbeat +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml index 650ec03920..3e4b364021 100644 --- a/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-hv-ves-collector/values.yaml @@ -22,6 +22,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat configuration defaults. @@ -45,8 +46,11 @@ image: onap/org.onap.dcaegen2.collectors.hv-ves.hv-collector-main:1.9.1 pullPolicy: Always # log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcae-hv-ves-collector +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcae-hv-ves-collector +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-kpi-ms/values.yaml b/kubernetes/dcaegen2-services/components/dcae-kpi-ms/values.yaml index 5d8c6d59eb..037dd0aec0 100644 --- a/kubernetes/dcaegen2-services/components/dcae-kpi-ms/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-kpi-ms/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -53,8 +54,11 @@ image: onap/org.onap.dcaegen2.services.components.kpi-ms:1.0.1 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/kpims +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/kpims +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml b/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml index caae1c319e..39c4a8ed50 100644 --- a/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-pm-mapper/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -58,8 +59,11 @@ image: onap/org.onap.dcaegen2.services.pm-mapper:1.7.2 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/pm-mapper +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/pm-mapper +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-pmsh/values.yaml b/kubernetes/dcaegen2-services/components/dcae-pmsh/values.yaml index b9005f01b8..4bdd2b8088 100644 --- a/kubernetes/dcaegen2-services/components/dcae-pmsh/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-pmsh/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -60,8 +61,11 @@ image: onap/org.onap.dcaegen2.services.pmsh:1.3.2 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/pmsh +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/pmsh +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-prh/values.yaml b/kubernetes/dcaegen2-services/components/dcae-prh/values.yaml index a7f62912b1..a4ed6994f7 100644 --- a/kubernetes/dcaegen2-services/components/dcae-prh/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-prh/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat configuration defaults. @@ -43,8 +44,11 @@ image: onap/org.onap.dcaegen2.services.prh.prh-app-server:1.7.1 pullPolicy: Always # log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /opt/app/prh/logs +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /opt/app/prh/logs +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-restconf-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-restconf-collector/values.yaml index 66c781cb3b..a4d8ecb1c3 100644 --- a/kubernetes/dcaegen2-services/components/dcae-restconf-collector/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-restconf-collector/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: false ################################################################# # Filebeat Configuration Defaults. @@ -54,8 +55,11 @@ image: onap/org.onap.dcaegen2.collectors.restconfcollector:1.2.5 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -# logDirectory: /opt/app/restconfcollector/logs +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: null # /opt/app/restconfcollector/logs +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-slice-analysis-ms/values.yaml b/kubernetes/dcaegen2-services/components/dcae-slice-analysis-ms/values.yaml index 3300306668..6cebca6412 100644 --- a/kubernetes/dcaegen2-services/components/dcae-slice-analysis-ms/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-slice-analysis-ms/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -59,8 +60,11 @@ consulLoaderImage: onap/org.onap.dcaegen2.deployments.consul-loader-container:1. image: onap/org.onap.dcaegen2.services.components.slice-analysis-ms:1.0.6 # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/sliceanalysisms +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/sliceanalysisms +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-snmptrap-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-snmptrap-collector/values.yaml index 266da24f7a..e2501729ec 100644 --- a/kubernetes/dcaegen2-services/components/dcae-snmptrap-collector/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-snmptrap-collector/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -44,8 +45,11 @@ image: onap/org.onap.dcaegen2.collectors.snmptrap:2.0.5 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /opt/app/snmptrap/logs +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /opt/app/snmptrap/logs +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # TLS role -- set to true if microservice acts as server diff --git a/kubernetes/dcaegen2-services/components/dcae-son-handler/values.yaml b/kubernetes/dcaegen2-services/components/dcae-son-handler/values.yaml index 54dcda831e..94c4d880dd 100644 --- a/kubernetes/dcaegen2-services/components/dcae-son-handler/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-son-handler/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -65,8 +66,11 @@ image: onap/org.onap.dcaegen2.services.son-handler:2.1.5 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /var/log/ONAP/dcaegen2/services/sonhms +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /var/log/ONAP/dcaegen2/services/sonhms +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-tcagen2/values.yaml b/kubernetes/dcaegen2-services/components/dcae-tcagen2/values.yaml index 2ba8edbc68..a65fa7c347 100644 --- a/kubernetes/dcaegen2-services/components/dcae-tcagen2/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-tcagen2/values.yaml @@ -22,6 +22,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat configuration defaults. @@ -45,8 +46,11 @@ image: onap/org.onap.dcaegen2.analytics.tca-gen2.dcae-analytics-tca-web:1.3.1 pullPolicy: Always # log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /opt/logs/dcae-analytics-tca +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /opt/logs/dcae-analytics-tca +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-ves-collector/values.yaml b/kubernetes/dcaegen2-services/components/dcae-ves-collector/values.yaml index 31007f2dde..e65a293d23 100644 --- a/kubernetes/dcaegen2-services/components/dcae-ves-collector/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-ves-collector/values.yaml @@ -22,6 +22,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat configuration defaults. @@ -45,8 +46,11 @@ image: onap/org.onap.dcaegen2.collectors.ves.vescollector:1.10.1 pullPolicy: Always # log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /opt/app/VESCollector/logs +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /opt/app/VESCollector/logs +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/components/dcae-ves-mapper/values.yaml b/kubernetes/dcaegen2-services/components/dcae-ves-mapper/values.yaml index 93214d3057..82826bca06 100644 --- a/kubernetes/dcaegen2-services/components/dcae-ves-mapper/values.yaml +++ b/kubernetes/dcaegen2-services/components/dcae-ves-mapper/values.yaml @@ -21,6 +21,7 @@ global: nodePortPrefix: 302 nodePortPrefixExt: 304 + centralizedLoggingEnabled: true ################################################################# # Filebeat Configuration Defaults. @@ -43,8 +44,11 @@ image: onap/org.onap.dcaegen2.services.mapper.vesadapter.universalvesadaptor:1.3 pullPolicy: Always # Log directory where logging sidecar should look for log files -# if absent, no sidecar will be deployed -logDirectory: /opt/app/VESAdapter/logs +# if path is set to null sidecar won't be deployed in spite of +# global.centralizedLoggingEnabled setting. +log: + path: /opt/app/VESAdapter/logs +logConfigMapNamePrefix: '{{ include "common.fullname" . }}' # Directory where TLS certs should be stored # if absent, no certs will be retrieved and stored diff --git a/kubernetes/dcaegen2-services/resources/config/log/filebeat/filebeat.yml b/kubernetes/dcaegen2-services/resources/config/log/filebeat/filebeat.yml new file mode 100644 index 0000000000..af62dc30ca --- /dev/null +++ b/kubernetes/dcaegen2-services/resources/config/log/filebeat/filebeat.yml @@ -0,0 +1,84 @@ +{{/* +#============LICENSE_START======================================================== +# ================================================================================ +# Copyright (c) 2018 AT&T Intellectual Property. All rights reserved. +# Modifications Copyright © 2018 Amdocs, Bell Canada +# Copyright (c) 2021 J. F. Lucas. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============LICENSE_END========================================================= +*/}} +{{/* +dcaegen2-services-common.filebeatConfiguration: +This template generates configuration data for filebeat (log file aggregation). + +The template is used to create a configMap mounted by a filebeat sidecar pod +running alongside a DCAE microservice pod. + +See dcaegen2-services-common.configMap for more information. +*/}} +filebeat.prospectors: +#it is mandatory, in our case it's log +- input_type: log + #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. + paths: + - /var/log/onap/*/*/*/*.log + - /var/log/onap/*/*/*.log + - /var/log/onap/*/*.log + #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive + ignore_older: 48h + # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit + clean_inactive: 96h + + +# Name of the registry file. If a relative path is used, it is considered relative to the +# data path. Else full qualified file name. +#filebeat.registry_file: ${path.data}/registry + + +output.logstash: + #List of logstash server ip addresses with port number. + #But, in our case, this will be the loadbalancer IP address. + #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. + hosts: ["{{.Values.filebeatConfig.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.filebeatConfig.logstashPort}}"] + #If enable will do load balancing among availabe Logstash, automatically. + loadbalance: true + + #The list of root certificates for server verifications. + #If certificate_authorities is empty or not set, the trusted + #certificate authorities of the host system are used. + #ssl.certificate_authorities: $ssl.certificate_authorities + + #The path to the certificate for SSL client authentication. If the certificate is not specified, + #client authentication is not available. + #ssl.certificate: $ssl.certificate + + #The client certificate key used for client authentication. + #ssl.key: $ssl.key + + #The passphrase used to decrypt an encrypted key stored in the configured key file + #ssl.key_passphrase: $ssl.key_passphrase + +logging: + level: debug + + # enable file rotation with default configuration + to_files: true + + # do not log to syslog + to_syslog: false + + files: + path: /usr/share/filebeat/logs + name: mybeat.log + keepfiles: 7 diff --git a/kubernetes/dcaegen2-services/templates/configmap.yaml b/kubernetes/dcaegen2-services/templates/configmap.yaml index 798f2a7939..52eb39fbf8 100644 --- a/kubernetes/dcaegen2-services/templates/configmap.yaml +++ b/kubernetes/dcaegen2-services/templates/configmap.yaml @@ -40,4 +40,6 @@ metadata: name: {{ include "common.release" . }}-dcae-external-repo-configmap-sa88-rel16 namespace: {{ include "common.namespace" . }} data: -{{ (.Files.Glob "resources/external/schemas/sa88-rel16/*").AsConfig | indent 2 }} \ No newline at end of file +{{ (.Files.Glob "resources/external/schemas/sa88-rel16/*").AsConfig | indent 2 }} +--- +{{ include "common.log.configMap" . }} diff --git a/kubernetes/dcaegen2-services/values.yaml b/kubernetes/dcaegen2-services/values.yaml index eb0b9cc34d..7db369c43f 100644 --- a/kubernetes/dcaegen2-services/values.yaml +++ b/kubernetes/dcaegen2-services/values.yaml @@ -14,43 +14,73 @@ # See the License for the specific language governing permissions and # limitations under the License. +global: + centralizedLoggingEnabled: true + +################################################################# +# Filebeat Configuration Defaults. +# +################################################################# +filebeatConfig: + logstashServiceName: log-ls + logstashPort: 5044 + # Control deployment of DCAE microservices at ONAP installation time dcae-bbs-eventprocessor-ms: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-datafile-collector: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-datalake-admin-ui: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-datalake-des: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-datalake-feeder: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-heartbeat: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-hv-ves-collector: enabled: true + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-kpi-ms: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-ms-healthcheck: enabled: true + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-pm-mapper: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-pmsh: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-prh: enabled: true + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-restconf-collector: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-slice-analysis-ms: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-snmptrap-collector: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-son-handler: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-tcagen2: enabled: true + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-ves-collector: enabled: true + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' dcae-ves-mapper: enabled: false + logConfigMapNamePrefix: '{{ include "common.release" . }}-dcaegen2-services' -- cgit 1.2.3-korg