From 172c0860cffb5fdfa2fc5e2348193da852ee1376 Mon Sep 17 00:00:00 2001 From: Maciej Wereski Date: Mon, 15 Nov 2021 11:34:06 +0100 Subject: [MODELING] Use log template Ability to turn off filebeat is needed as it is being deprecated. To achieve that existing log helper template is used. Issue-ID: OOM-1 Signed-off-by: Maciej Wereski Change-Id: Ifb21cc0a10e1ac2770e6835ad883b5d0f0f1735d --- .../resources/config/log/filebeat/filebeat.yml | 41 ++++++++++++++++ .../resources/config/log/server/log.yml | 54 ++++++++++++++++++++++ .../resources/config/logging/filebeat/filebeat.yml | 41 ---------------- .../resources/config/logging/server/log.yml | 54 ---------------------- .../modeling-etsicatalog/templates/configmap.yaml | 10 +--- .../modeling-etsicatalog/templates/deployment.yaml | 20 ++------ .../components/modeling-etsicatalog/values.yaml | 6 +++ 7 files changed, 106 insertions(+), 120 deletions(-) create mode 100644 kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/filebeat/filebeat.yml create mode 100644 kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/server/log.yml delete mode 100644 kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/filebeat/filebeat.yml delete mode 100644 kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/server/log.yml diff --git a/kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/filebeat/filebeat.yml b/kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/filebeat/filebeat.yml new file mode 100644 index 0000000000..0bc14ea908 --- /dev/null +++ b/kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/filebeat/filebeat.yml @@ -0,0 +1,41 @@ +filebeat.prospectors: +#it is mandatory, in our case it's log +- input_type: log + #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. + paths: + - /var/log/onap/*/*/*/*.log + - /var/log/onap/*/*/*.log + - /var/log/onap/*/*.log + #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive + ignore_older: 48h + # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit + clean_inactive: 96h + + +# Name of the registry file. If a relative path is used, it is considered relative to the +# data path. Else full qualified file name. +#filebeat.registry_file: ${path.data}/registry + + +output.logstash: + #List of logstash server ip addresses with port number. + #But, in our case, this will be the loadbalancer IP address. + #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. + hosts: ["{{.Values.config.logstashServiceName}}:{{.Values.config.logstashPort}}"] + #If enable will do load balancing among availabe Logstash, automatically. + loadbalance: true + + #The list of root certificates for server verifications. + #If certificate_authorities is empty or not set, the trusted + #certificate authorities of the host system are used. + #ssl.certificate_authorities: $ssl.certificate_authorities + + #The path to the certificate for SSL client authentication. If the certificate is not specified, + #client authentication is not available. + #ssl.certificate: $ssl.certificate + + #The client certificate key used for client authentication. + #ssl.key: $ssl.key + + #The passphrase used to decrypt an encrypted key stored in the configured key file + #ssl.key_passphrase: $ssl.key_passphrase diff --git a/kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/server/log.yml b/kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/server/log.yml new file mode 100644 index 0000000000..5ac5fefe92 --- /dev/null +++ b/kubernetes/modeling/components/modeling-etsicatalog/resources/config/log/server/log.yml @@ -0,0 +1,54 @@ +version: 1 +disable_existing_loggers: False + +loggers: + catalog: + handlers: [cataloglocal_handler, catalog_handler] + level: "DEBUG" + propagate: False + django: + handlers: [django_handler] + level: "DEBUG" + propagate: False + tosca: + handlers: [cataloglocal_handler, catalog_handler] + level: "DEBUG" + propagate: False +handlers: + cataloglocal_handler: + level: "DEBUG" + class: + "logging.handlers.RotatingFileHandler" + filename: "logs/runtime_catalog.log" + formatter: + "standard" + maxBytes: 52428800 + backupCount: 10 + catalog_handler: + level: "DEBUG" + class: + "logging.handlers.RotatingFileHandler" + filename: "/var/log/onap/modeling/etsicatalog/runtime_catalog.log" + formatter: + "mdcFormat" + maxBytes: 52428800 + backupCount: 10 + django_handler: + level: "DEBUG" + class: + "logging.handlers.RotatingFileHandler" + filename: "logs/django.log" + formatter: + "standard" + maxBytes: 52428800 + backupCount: 10 +formatters: + standard: + format: + "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" + mdcFormat: + format: + "%(asctime)s|||||%(name)s||%(thread)s||%(funcName)s||%(levelname)s||%(message)s||||%(mdc)s \t" + mdcfmt: "{requestID} {invocationID} {serviceName} {serviceIP}" + datefmt: "%Y-%m-%d %H:%M:%S" + (): onaplogging.mdcformatter.MDCFormatter diff --git a/kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/filebeat/filebeat.yml b/kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/filebeat/filebeat.yml deleted file mode 100644 index 0bc14ea908..0000000000 --- a/kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/filebeat/filebeat.yml +++ /dev/null @@ -1,41 +0,0 @@ -filebeat.prospectors: -#it is mandatory, in our case it's log -- input_type: log - #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. - paths: - - /var/log/onap/*/*/*/*.log - - /var/log/onap/*/*/*.log - - /var/log/onap/*/*.log - #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive - ignore_older: 48h - # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit - clean_inactive: 96h - - -# Name of the registry file. If a relative path is used, it is considered relative to the -# data path. Else full qualified file name. -#filebeat.registry_file: ${path.data}/registry - - -output.logstash: - #List of logstash server ip addresses with port number. - #But, in our case, this will be the loadbalancer IP address. - #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. - hosts: ["{{.Values.config.logstashServiceName}}:{{.Values.config.logstashPort}}"] - #If enable will do load balancing among availabe Logstash, automatically. - loadbalance: true - - #The list of root certificates for server verifications. - #If certificate_authorities is empty or not set, the trusted - #certificate authorities of the host system are used. - #ssl.certificate_authorities: $ssl.certificate_authorities - - #The path to the certificate for SSL client authentication. If the certificate is not specified, - #client authentication is not available. - #ssl.certificate: $ssl.certificate - - #The client certificate key used for client authentication. - #ssl.key: $ssl.key - - #The passphrase used to decrypt an encrypted key stored in the configured key file - #ssl.key_passphrase: $ssl.key_passphrase diff --git a/kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/server/log.yml b/kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/server/log.yml deleted file mode 100644 index 5ac5fefe92..0000000000 --- a/kubernetes/modeling/components/modeling-etsicatalog/resources/config/logging/server/log.yml +++ /dev/null @@ -1,54 +0,0 @@ -version: 1 -disable_existing_loggers: False - -loggers: - catalog: - handlers: [cataloglocal_handler, catalog_handler] - level: "DEBUG" - propagate: False - django: - handlers: [django_handler] - level: "DEBUG" - propagate: False - tosca: - handlers: [cataloglocal_handler, catalog_handler] - level: "DEBUG" - propagate: False -handlers: - cataloglocal_handler: - level: "DEBUG" - class: - "logging.handlers.RotatingFileHandler" - filename: "logs/runtime_catalog.log" - formatter: - "standard" - maxBytes: 52428800 - backupCount: 10 - catalog_handler: - level: "DEBUG" - class: - "logging.handlers.RotatingFileHandler" - filename: "/var/log/onap/modeling/etsicatalog/runtime_catalog.log" - formatter: - "mdcFormat" - maxBytes: 52428800 - backupCount: 10 - django_handler: - level: "DEBUG" - class: - "logging.handlers.RotatingFileHandler" - filename: "logs/django.log" - formatter: - "standard" - maxBytes: 52428800 - backupCount: 10 -formatters: - standard: - format: - "%(asctime)s:[%(name)s]:[%(filename)s]-[%(lineno)d] [%(levelname)s]:%(message)s" - mdcFormat: - format: - "%(asctime)s|||||%(name)s||%(thread)s||%(funcName)s||%(levelname)s||%(message)s||||%(mdc)s \t" - mdcfmt: "{requestID} {invocationID} {serviceName} {serviceIP}" - datefmt: "%Y-%m-%d %H:%M:%S" - (): onaplogging.mdcformatter.MDCFormatter diff --git a/kubernetes/modeling/components/modeling-etsicatalog/templates/configmap.yaml b/kubernetes/modeling/components/modeling-etsicatalog/templates/configmap.yaml index 1a2112bbe2..fc16d8c0a0 100644 --- a/kubernetes/modeling/components/modeling-etsicatalog/templates/configmap.yaml +++ b/kubernetes/modeling/components/modeling-etsicatalog/templates/configmap.yaml @@ -20,12 +20,6 @@ metadata: name: {{ include "common.fullname" . }}-logging-configmap namespace: {{ include "common.namespace" . }} data: -{{ tpl (.Files.Glob "resources/config/logging/server/*").AsConfig . | indent 2 }} +{{ tpl (.Files.Glob "resources/config/log/server/*").AsConfig . | indent 2 }} --- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "common.fullname" . }}-modeling-filebeat-configmap - namespace: {{ include "common.namespace" . }} -data: -{{ tpl (.Files.Glob "resources/config/logging/filebeat/*").AsConfig . | indent 2 }} \ No newline at end of file +{{ include "common.log.configMap" . }} diff --git a/kubernetes/modeling/components/modeling-etsicatalog/templates/deployment.yaml b/kubernetes/modeling/components/modeling-etsicatalog/templates/deployment.yaml index 8e28071ba6..d9cbbef15b 100644 --- a/kubernetes/modeling/components/modeling-etsicatalog/templates/deployment.yaml +++ b/kubernetes/modeling/components/modeling-etsicatalog/templates/deployment.yaml @@ -111,7 +111,7 @@ spec: mountPath: /etc/localtime readOnly: true - name: {{ include "common.fullname" . }}-logs - mountPath: /var/log/onap + mountPath: {{ .Values.log.path }} - name: {{ include "common.fullname" . }}-logconfig mountPath: /opt/modeling/etsicatalog/config/log.yml subPath: log.yml @@ -127,17 +127,7 @@ spec: {{- end }} # side car containers - - name: {{ include "common.name" . }}-filebeat-onap - image: {{ include "repositoryGenerator.image.logging" . }} - imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} - volumeMounts: - - name: {{ include "common.fullname" . }}-filebeat-conf - mountPath: /usr/share/filebeat/filebeat.yml - subPath: filebeat.yml - - name: {{ include "common.fullname" . }}-logs - mountPath: /var/log/onap - - name: {{ include "common.fullname" . }}-data-filebeat - mountPath: /usr/share/filebeat/data + {{ include "common.log.sidecar" . | nindent 8 }} serviceAccountName: {{ include "common.fullname" (dict "suffix" "read" "dot" . )}} volumes: - name: {{ include "common.fullname" . }}-etsicatalog @@ -156,10 +146,6 @@ spec: configMap: name : {{ include "common.fullname" . }}-logging-configmap - - name: {{ include "common.fullname" . }}-filebeat-conf - configMap: - name: {{ include "common.fullname" . }}-modeling-filebeat-configmap - - name: {{ include "common.fullname" . }}-data-filebeat - emptyDir: {} + {{ include "common.log.volumes" . | nindent 8 }} imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/modeling/components/modeling-etsicatalog/values.yaml b/kubernetes/modeling/components/modeling-etsicatalog/values.yaml index 3af4bae554..b4d88375fc 100644 --- a/kubernetes/modeling/components/modeling-etsicatalog/values.yaml +++ b/kubernetes/modeling/components/modeling-etsicatalog/values.yaml @@ -28,6 +28,8 @@ global: internalPort: 3306 nameOverride: mariadb-galera + centralizedLoggingEnabled: true + readinessCheck: wait_for: msb: @@ -183,3 +185,7 @@ serviceAccount: nameOverride: modeling-etsicatalog roles: - read + +#Log configuration +log: + path: /var/log/onap -- cgit 1.2.3-korg