From 531d03fe9bbcff204520eb4e7bb606bc20a50cbb Mon Sep 17 00:00:00 2001 From: Sylvain Desbureaux Date: Wed, 22 Apr 2020 10:37:45 +0200 Subject: [APPC] use common templates for logs Using the common templates for centralized logging. Issue-ID: OOM-2370 Signed-off-by: Sylvain Desbureaux Change-Id: Ib0995b754597da36d06939199738cf93398e796a --- .../resources/config/log/filebeat/filebeat.yml | 63 ++++++++++++++++++++++ .../config/log/filebeat/log4j/filebeat.yml | 63 ---------------------- kubernetes/appc/templates/configmap.yaml | 15 +----- kubernetes/appc/templates/statefulset.yaml | 20 ++----- kubernetes/appc/values.yaml | 5 ++ 5 files changed, 73 insertions(+), 93 deletions(-) create mode 100644 kubernetes/appc/resources/config/log/filebeat/filebeat.yml delete mode 100644 kubernetes/appc/resources/config/log/filebeat/log4j/filebeat.yml diff --git a/kubernetes/appc/resources/config/log/filebeat/filebeat.yml b/kubernetes/appc/resources/config/log/filebeat/filebeat.yml new file mode 100644 index 0000000000..85293c8275 --- /dev/null +++ b/kubernetes/appc/resources/config/log/filebeat/filebeat.yml @@ -0,0 +1,63 @@ +# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +filebeat.prospectors: +#it is mandatory, in our case it's log +- input_type: log + #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. + paths: + - /var/log/onap/*/*/*/*.log + - /var/log/onap/*/*/*.log + - /var/log/onap/*/*.log + + # The below commented properties are for time-based rolling policy. But as the log4j 1.2x does not support time-based rolling these properties are not set + #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive + #ignore_older: 48h + # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit + #clean_inactive: 96h + + #Multiline properties for log4j xml log events + multiline.pattern: '' + multiline.negate: true + multiline.match: before + #multiline.max_lines: 500 + #multiline.timeout: 5s + +# Name of the registry file. If a relative path is used, it is considered relative to the +# data path. Else full qualified file name. +#filebeat.registry_file: ${path.data}/registry + + +output.logstash: + #List of logstash server ip addresses with port number. + #But, in our case, this will be the loadbalancer IP address. + #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. + hosts: ["{{.Values.config.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.config.logstashPort}}"] + #If enable will do load balancing among availabe Logstash, automatically. + loadbalance: true + + #The list of root certificates for server verifications. + #If certificate_authorities is empty or not set, the trusted + #certificate authorities of the host system are used. + #ssl.certificate_authorities: $ssl.certificate_authorities + + #The path to the certificate for SSL client authentication. If the certificate is not specified, + #client authentication is not available. + #ssl.certificate: $ssl.certificate + + #The client certificate key used for client authentication. + #ssl.key: $ssl.key + + #The passphrase used to decrypt an encrypted key stored in the configured key file + #ssl.key_passphrase: $ssl.key_passphrase diff --git a/kubernetes/appc/resources/config/log/filebeat/log4j/filebeat.yml b/kubernetes/appc/resources/config/log/filebeat/log4j/filebeat.yml deleted file mode 100644 index 85293c8275..0000000000 --- a/kubernetes/appc/resources/config/log/filebeat/log4j/filebeat.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -filebeat.prospectors: -#it is mandatory, in our case it's log -- input_type: log - #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. - paths: - - /var/log/onap/*/*/*/*.log - - /var/log/onap/*/*/*.log - - /var/log/onap/*/*.log - - # The below commented properties are for time-based rolling policy. But as the log4j 1.2x does not support time-based rolling these properties are not set - #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive - #ignore_older: 48h - # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit - #clean_inactive: 96h - - #Multiline properties for log4j xml log events - multiline.pattern: '' - multiline.negate: true - multiline.match: before - #multiline.max_lines: 500 - #multiline.timeout: 5s - -# Name of the registry file. If a relative path is used, it is considered relative to the -# data path. Else full qualified file name. -#filebeat.registry_file: ${path.data}/registry - - -output.logstash: - #List of logstash server ip addresses with port number. - #But, in our case, this will be the loadbalancer IP address. - #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. - hosts: ["{{.Values.config.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.config.logstashPort}}"] - #If enable will do load balancing among availabe Logstash, automatically. - loadbalance: true - - #The list of root certificates for server verifications. - #If certificate_authorities is empty or not set, the trusted - #certificate authorities of the host system are used. - #ssl.certificate_authorities: $ssl.certificate_authorities - - #The path to the certificate for SSL client authentication. If the certificate is not specified, - #client authentication is not available. - #ssl.certificate: $ssl.certificate - - #The client certificate key used for client authentication. - #ssl.key: $ssl.key - - #The passphrase used to decrypt an encrypted key stored in the configured key file - #ssl.key_passphrase: $ssl.key_passphrase diff --git a/kubernetes/appc/templates/configmap.yaml b/kubernetes/appc/templates/configmap.yaml index 72dc6172dc..fe206a9322 100644 --- a/kubernetes/appc/templates/configmap.yaml +++ b/kubernetes/appc/templates/configmap.yaml @@ -118,19 +118,6 @@ data: --- apiVersion: v1 kind: ConfigMap -metadata: - name: {{ include "common.fullname" . }}-filebeat - namespace: {{ include "common.namespace" . }} - labels: - app: {{ include "common.name" . }} - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ include "common.release" . }} - heritage: {{ .Release.Service }} -data: -{{ tpl (.Files.Glob "resources/config/log/filebeat/log4j/*").AsConfig . | indent 2 }} ---- -apiVersion: v1 -kind: ConfigMap metadata: name: {{ include "common.fullname" . }}-logging-cfg namespace: {{ include "common.namespace" . }} @@ -141,3 +128,5 @@ metadata: heritage: {{ .Release.Service }} data: {{ tpl (.Files.Glob "resources/config/log/*").AsConfig . | indent 2 }} + +{{ include "common.log.configMap" . }} diff --git a/kubernetes/appc/templates/statefulset.yaml b/kubernetes/appc/templates/statefulset.yaml index 5fd34ece79..3480e638aa 100644 --- a/kubernetes/appc/templates/statefulset.yaml +++ b/kubernetes/appc/templates/statefulset.yaml @@ -185,7 +185,7 @@ spec: subPath: installSdncDb.sh - mountPath: {{ .Values.persistence.mdsalPath }} name: {{ include "common.fullname" . }}-data - - mountPath: /var/log/onap + - mountPath: {{ .Values.log.path }} name: logs - mountPath: /opt/onap/appc/data/org.ops4j.pax.logging.cfg name: log-config @@ -208,17 +208,7 @@ spec: {{- end }} # side car containers - - name: filebeat-onap - image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}" - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - volumeMounts: - - mountPath: /usr/share/filebeat/filebeat.yml - name: filebeat-conf - subPath: filebeat.yml - - mountPath: /var/log/onap - name: logs - - mountPath: /usr/share/filebeat/data - name: data-filebeat + {{ include "common.log.sidecar" . | nindent 8 }} volumes: - name: keyfile-certs secret: @@ -229,16 +219,12 @@ spec: - name: localtime hostPath: path: /etc/localtime - - name: filebeat-conf - configMap: - name: {{ include "common.fullname" . }}-filebeat - name: log-config configMap: name: {{ include "common.fullname" . }}-logging-cfg - name: logs emptyDir: {} - - name: data-filebeat - emptyDir: {} + {{ include "common.log.volumes" . | nindent 8 }} - name: onap-appc-data-properties-input configMap: name: {{ include "common.fullname" . }}-onap-appc-data-properties diff --git a/kubernetes/appc/values.yaml b/kubernetes/appc/values.yaml index fb8230b128..cde892002b 100644 --- a/kubernetes/appc/values.yaml +++ b/kubernetes/appc/values.yaml @@ -20,6 +20,7 @@ global: nodePortPrefix: 302 readinessRepository: oomk8s readinessImage: readiness-check:2.0.0 + centralizedLoggingEnabled: false loggingRepository: docker.elastic.co loggingImage: beats/filebeat:5.5.0 # envsusbt @@ -62,6 +63,10 @@ pullPolicy: Always # flag to enable debugging - application support required debugEnabled: false +# log configuration +log: + path: /var/log/onap + # application configuration config: # dbRootPassExternalSecret: some secret -- cgit 1.2.3-korg