From 378cf19f543cda153333273bd348c7092c5898bb Mon Sep 17 00:00:00 2001 From: econwar Date: Tue, 8 Jan 2019 16:19:59 +0000 Subject: Add filebeat sidecar container for dmaap-dr Change-Id: I3f81a683832ec37598e165c859944ab97a9ffc51 Issue-ID: DMAAP-853 Signed-off-by: econwar --- .../dmaap-dr-node/resources/config/logback.xml | 148 +++++++++++++++++++++ .../charts/dmaap-dr-node/templates/configmap.yaml | 8 ++ .../charts/dmaap-dr-node/templates/deployment.yaml | 25 ++++ .../dmaap-dr-prov/resources/config/logback.xml | 148 +++++++++++++++++++++ .../charts/dmaap-dr-prov/templates/configmap.yaml | 8 ++ .../charts/dmaap-dr-prov/templates/deployment.yaml | 25 ++++ .../dmaap/charts/dmaap-data-router/values.yaml | 1 + .../resources/config/log/filebeat/filebeat.yml | 60 +++++++++ kubernetes/dmaap/templates/configmap.yaml | 26 ++++ kubernetes/dmaap/values.yaml | 4 + 10 files changed, 453 insertions(+) create mode 100644 kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/resources/config/logback.xml create mode 100644 kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/resources/config/logback.xml create mode 100644 kubernetes/dmaap/resources/config/log/filebeat/filebeat.yml create mode 100644 kubernetes/dmaap/templates/configmap.yaml diff --git a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/resources/config/logback.xml b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/resources/config/logback.xml new file mode 100644 index 0000000000..25a2c6e093 --- /dev/null +++ b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/resources/config/logback.xml @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + ${defaultPattern} + + + + + + + + + ${logDirectory}/${generalLogName}.log + + INFO + ACCEPT + DENY + + + ${logDirectory}/${generalLogName}.%i.log.zip + + 1 + 9 + + + 5MB + + + ${defaultPattern} + + + + + 256 + + + + + ${logDirectory}/${errorLogName}.log + + ERROR + ACCEPT + DENY + + + ${logDirectory}/${errorLogName}.%i.log.zip + + 1 + 9 + + + 5MB + + + ${defaultPattern} + + + + + 256 + + + + + + ${logDirectory}/${jettyAndNodeLogName}.log + + + ${logDirectory}/${jettyAndNodeLogName}.%i.log.zip + + 1 + 9 + + + 5MB + + + ${jettyAndNodeLoggerPattern} + + + + + 256 + + true + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/configmap.yaml b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/configmap.yaml index bd7418415a..67392671f8 100644 --- a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/configmap.yaml +++ b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/configmap.yaml @@ -37,3 +37,11 @@ metadata: heritage: {{ .Release.Service }} data: {{ tpl (.Files.Glob "resources/config/feeds/*").AsConfig . | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-log + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/logback.xml").AsConfig . | indent 2 }} diff --git a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/deployment.yaml b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/deployment.yaml index 93fabe1292..a06b3854c3 100644 --- a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/deployment.yaml +++ b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-node/templates/deployment.yaml @@ -77,6 +77,8 @@ spec: - mountPath: /opt/app/datartr/etc/node.properties subPath: node.properties name: node-props + - name: {{ include "common.fullname" . }}-logs + mountPath: {{ .Values.global.loggingDirectory }} lifecycle: postStart: exec: @@ -92,6 +94,19 @@ spec: affinity: {{ toYaml .Values.affinity | indent 10 }} {{- end }} + + # Filebeat sidecar container + - name: {{ include "common.name" . }}-filebeat-onap + image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + volumeMounts: + - name: {{ include "common.fullname" . }}-filebeat-conf + mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + - name: {{ include "common.fullname" . }}-data-filebeat + mountPath: /usr/share/filebeat/data + - name: {{ include "common.fullname" . }}-logs + mountPath: /var/log/onap/datarouter-node volumes: - name: localtime hostPath: @@ -110,5 +125,15 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: {{ include "common.fullname" . }}-log-conf + configMap: + name: {{ include "common.fullname" . }}-log + - name: {{ include "common.fullname" . }}-filebeat-conf + configMap: + name: {{ .Release.Name }}-dmaap-filebeat-configmap + - name: {{ include "common.fullname" . }}-data-filebeat + emptyDir: {} + - name: {{ include "common.fullname" . }}-logs + emptyDir: {} imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/resources/config/logback.xml b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/resources/config/logback.xml new file mode 100644 index 0000000000..942d898695 --- /dev/null +++ b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/resources/config/logback.xml @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + ${defaultPattern} + + + + + + + + + ${logDirectory}/${generalLogName}.log + + INFO + ACCEPT + DENY + + + ${logDirectory}/${generalLogName}.%i.log.zip + + 1 + 9 + + + 5MB + + + ${defaultPattern} + + + + + 256 + + + + + ${logDirectory}/${errorLogName}.log + + ERROR + ACCEPT + DENY + + + ${logDirectory}/${errorLogName}.%i.log.zip + + 1 + 9 + + + 5MB + + + ${defaultPattern} + + + + + 256 + + + + + + ${logDirectory}/${jettyLogName}.log + + + ${logDirectory}/${jettyLogName}.%i.log.zip + + 1 + 9 + + + 5MB + + + ${jettyLoggerPattern} + + + + + 256 + + true + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/configmap.yaml b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/configmap.yaml index 525c18b6ba..f9884d7827 100644 --- a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/configmap.yaml +++ b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/configmap.yaml @@ -24,3 +24,11 @@ metadata: heritage: {{ .Release.Service }} data: {{ tpl (.Files.Glob "resources/config/provserver.properties").AsConfig . | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-log + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/logback.xml").AsConfig . | indent 2 }} diff --git a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/deployment.yaml b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/deployment.yaml index da65944701..c1035105b2 100644 --- a/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/deployment.yaml +++ b/kubernetes/dmaap/charts/dmaap-data-router/charts/dmaap-dr-prov/templates/deployment.yaml @@ -71,6 +71,8 @@ spec: - mountPath: /opt/app/datartr/etc/provserver.properties subPath: provserver.properties name: prov-props + - name: {{ include "common.fullname" . }}-logs + mountPath: {{ .Values.global.loggingDirectory }} resources: {{ include "common.resources" . | indent 12 }} {{- if .Values.nodeSelector }} @@ -81,6 +83,19 @@ spec: affinity: {{ toYaml .Values.affinity | indent 10 }} {{- end }} + + # Filebeat sidecar container + - name: {{ include "common.name" . }}-filebeat-onap + image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + volumeMounts: + - name: {{ include "common.fullname" . }}-filebeat-conf + mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + - name: {{ include "common.fullname" . }}-data-filebeat + mountPath: /usr/share/filebeat/data + - name: {{ include "common.fullname" . }}-logs + mountPath: /var/log/onap/datarouter-prov volumes: - name: localtime hostPath: @@ -95,5 +110,15 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: {{ include "common.fullname" . }}-log-conf + configMap: + name: {{ include "common.fullname" . }}-log + - name: {{ include "common.fullname" . }}-filebeat-conf + configMap: + name: {{ .Release.Name }}-dmaap-filebeat-configmap + - name: {{ include "common.fullname" . }}-data-filebeat + emptyDir: {} + - name: {{ include "common.fullname" . }}-logs + emptyDir: {} imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/dmaap/charts/dmaap-data-router/values.yaml b/kubernetes/dmaap/charts/dmaap-data-router/values.yaml index 1a4feec659..c783d0745a 100644 --- a/kubernetes/dmaap/charts/dmaap-data-router/values.yaml +++ b/kubernetes/dmaap/charts/dmaap-data-router/values.yaml @@ -20,6 +20,7 @@ global: name: AUTO ubuntuInitRepository: oomk8s ubuntuInitImage: ubuntu-init:1.0.0 + loggingDirectory: /opt/app/datartr/logs config: # dr provisioning server configuration diff --git a/kubernetes/dmaap/resources/config/log/filebeat/filebeat.yml b/kubernetes/dmaap/resources/config/log/filebeat/filebeat.yml new file mode 100644 index 0000000000..f1cf7394b6 --- /dev/null +++ b/kubernetes/dmaap/resources/config/log/filebeat/filebeat.yml @@ -0,0 +1,60 @@ +# ============LICENSE_START======================================================= +# Copyright (C) 2019 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +filebeat.prospectors: +#it is mandatory, in our case it's log +- input_type: log + #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. + paths: + - /var/log/onap/*/*/*/*.log + - /var/log/onap/*/*/*.log + - /var/log/onap/*/*.log + - /opt/app/datartr/logs/*.log + #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive + ignore_older: 48h + # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit + clean_inactive: 96h + + +# Name of the registry file. If a relative path is used, it is considered relative to the +# data path. Else full qualified file name. +#filebeat.registry_file: ${path.data}/registry + + +output.logstash: + #List of logstash server ip addresses with port number. + #But, in our case, this will be the loadbalancer IP address. + #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. + hosts: ["{{.Values.config.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.config.logstashPort}}"] + #If enable will do load balancing among available Logstash, automatically. + loadbalance: true + + #The list of root certificates for server verifications. + #If certificate_authorities is empty or not set, the trusted + #certificate authorities of the host system are used. + #ssl.certificate_authorities: $ssl.certificate_authorities + + #The path to the certificate for SSL client authentication. If the certificate is not specified, + #client authentication is not available. + #ssl.certificate: $ssl.certificate + + #The client certificate key used for client authentication. + #ssl.key: $ssl.key + + #The passphrase used to decrypt an encrypted key stored in the configured key file + #ssl.key_passphrase: $ssl.key_passphrase \ No newline at end of file diff --git a/kubernetes/dmaap/templates/configmap.yaml b/kubernetes/dmaap/templates/configmap.yaml new file mode 100644 index 0000000000..6d801024f4 --- /dev/null +++ b/kubernetes/dmaap/templates/configmap.yaml @@ -0,0 +1,26 @@ +# +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-dmaap-filebeat-configmap + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/log/filebeat/filebeat.yml").AsConfig . | indent 2 }} \ No newline at end of file diff --git a/kubernetes/dmaap/values.yaml b/kubernetes/dmaap/values.yaml index 6ded7d1d1e..8e33abff14 100644 --- a/kubernetes/dmaap/values.yaml +++ b/kubernetes/dmaap/values.yaml @@ -22,3 +22,7 @@ global: readinessImage: readiness-check:2.0.0 loggingRepository: docker.elastic.co loggingImage: beats/filebeat:5.5.0 +# application configuration +config: + logstashServiceName: log-ls + logstashPort: 5044 -- cgit 1.2.3-korg