diff options
| -rw-r--r-- | kubernetes/clamp/resources/config/log/filebeat/filebeat.yml | 41 | ||||
| -rw-r--r-- | kubernetes/clamp/resources/config/logback.xml | 236 | ||||
| -rw-r--r-- | kubernetes/clamp/templates/configmap.yaml | 8 | ||||
| -rw-r--r-- | kubernetes/clamp/templates/deployment.yaml | 29 | ||||
| -rw-r--r-- | kubernetes/clamp/values.yaml | 3 |
5 files changed, 317 insertions, 0 deletions
diff --git a/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml b/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml new file mode 100644 index 0000000000..44c0e6a8ac --- /dev/null +++ b/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml @@ -0,0 +1,41 @@ +filebeat.prospectors: +#it is mandatory, in our case it's log +- input_type: log + #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. + paths: + - /var/log/onap/*/*/*/*.log + - /var/log/onap/*/*/*.log + - /var/log/onap/*/*.log + #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive + ignore_older: 48h + # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit + clean_inactive: 96h + + +# Name of the registry file. If a relative path is used, it is considered relative to the +# data path. Else full qualified file name. +#filebeat.registry_file: ${path.data}/registry + + +output.logstash: + #List of logstash server ip addresses with port number. + #But, in our case, this will be the loadbalancer IP address. + #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. + hosts: ["{{.Values.config.log.logstashServiceName}}:{{.Values.config.log.logstashPort}}"] + #If enable will do load balancing among availabe Logstash, automatically. + loadbalance: true + + #The list of root certificates for server verifications. + #If certificate_authorities is empty or not set, the trusted + #certificate authorities of the host system are used. + #ssl.certificate_authorities: $ssl.certificate_authorities + + #The path to the certificate for SSL client authentication. If the certificate is not specified, + #client authentication is not available. + #ssl.certificate: $ssl.certificate + + #The client certificate key used for client authentication. + #ssl.key: $ssl.key + + #The passphrase used to decrypt an encrypted key stored in the configured key file + #ssl.key_passphrase: $ssl.key_passphrase diff --git a/kubernetes/clamp/resources/config/logback.xml b/kubernetes/clamp/resources/config/logback.xml new file mode 100644 index 0000000000..0a8f8d5f84 --- /dev/null +++ b/kubernetes/clamp/resources/config/logback.xml @@ -0,0 +1,236 @@ +<configuration scan="true" scanPeriod="10 seconds" debug="false"> + <jmxConfigurator /> + <property resource="application.properties" /> + <property name="logDirectory" value="/var/log/onap/clamp" /> + <!-- Example evaluator filter applied against console appender --> + <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> + <!-- filter class="ch.qos.logback.classic.filter.LevelFilter"> <level>ERROR</level> + <onMatch>ACCEPT</onMatch> <onMismatch>DENY</onMismatch> </filter --> + <!-- deny all events with a level below INFO, that is TRACE and DEBUG --> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + <encoder> + <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{1024} - %msg%n + </pattern> + </encoder> + </appender> + + <appender name="ERROR" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + <file>${logDirectory}/error.log</file> + <append>true</append> + <encoder> + <pattern>%date{"yyyy-MM-dd'T'HH:mm:ss,SSSXXX", UTC}|%X{RequestId}|%.20thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%.-5level|%X{ErrorCode}|%X{ErrorDescription}|%msg%n</pattern> + </encoder> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <maxIndex>10</maxIndex> + <FileNamePattern>${logDirectory}/error.%i.log.zip + </FileNamePattern> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>10MB</maxFileSize> + </triggeringPolicy> + </appender> + + <appender name="DEBUG" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/debug.log</file> + <append>true</append> + <encoder> + <pattern>%date{"yyyy-MM-dd'T'HH:mm:ss,SSSXXX",UTC}|%X{RequestId}|%.20thread|%.-5level|%logger{36}|%msg%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/debug.%d{yyyy-MM-dd}.%i.log.zip</fileNamePattern> + <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> + <maxFileSize>10MB</maxFileSize> + </timeBasedFileNamingAndTriggeringPolicy> + <maxHistory>5</maxHistory> + </rollingPolicy> + </appender> + + <appender name="AUDIT" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/audit.log</file> + <append>true</append> + <encoder> + <pattern>%X{BeginTimestamp}|%X{EndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%.20thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}|%msg%n + </pattern> + </encoder> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <maxIndex>10</maxIndex> + <FileNamePattern>${logDirectory}/audit.%i.log.zip + </FileNamePattern> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>10MB</maxFileSize> + </triggeringPolicy> + </appender> + <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="AUDIT" /> + </appender> + + <appender name="METRIC" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/metric.log</file> + <append>true</append> + <encoder> + <pattern>%X{BeginTimestamp}|%X{EndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%.20thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{TargetVirtualEntity}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}| + %msg%n</pattern> + </encoder> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <maxIndex>10</maxIndex> + <FileNamePattern>${logDirectory}/metric.%i.log.zip + </FileNamePattern> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>10MB</maxFileSize> + </triggeringPolicy> + </appender> + <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="METRIC" /> + </appender> + + <!-- SECURITY related loggers --> + <appender name="SECURITY" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/security.log</file> + <append>true</append> + <encoder> + <pattern>%X{BeginTimestamp}|%X{EndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%.20thread|%X{VirtualServerName}|%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDescription}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ElapsedTime}|%X{ServerFQDN}|%X{RemoteHost}|%X{ClassName}|%X{Unused}|%X{ProcessKey}|%X{CustomField1}|%X{CustomField2}|%X{CustomField3}|%X{CustomField4}|%msg%n + </pattern> + </encoder> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <maxIndex>10</maxIndex> + <FileNamePattern>${logDirectory}/security.%i.log.zip + </FileNamePattern> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>10MB</maxFileSize> + </triggeringPolicy> + </appender> + <appender name="asyncEELFSecurity" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="SECURITY" /> + </appender> + + <!-- CLDS related loggers --> + <logger name="org.onap.clamp.clds" level="INFO" additivity="true"> + <appender-ref ref="ERROR" /> + </logger> + + <!-- CLDS related loggers --> + <logger name="com.att.eelf.error" level="OFF" additivity="true"> + <appender-ref ref="ERROR" /> + </logger> + <!-- EELF related loggers --> + <logger name="com.att.eelf.audit" level="INFO" additivity="false"> + <appender-ref ref="asyncEELFAudit" /> + </logger> + <logger name="com.att.eelf.metrics" level="DEBUG" additivity="false"> + <appender-ref ref="asyncEELFMetrics" /> + </logger> + <logger name="com.att.eelf.security" level="DEBUG" additivity="false"> + <appender-ref ref="asyncEELFSecurity" /> + </logger> + + + <!-- Spring related loggers --> + <logger name="org.springframework" level="WARN" /> + <logger name="org.springframework.beans" level="WARN" /> + <logger name="org.springframework.web" level="WARN" /> + <logger name="com.blog.spring.jms" level="WARN" /> + + <!-- Other Loggers that may help troubleshoot --> + <logger name="net.sf" level="WARN" /> + <logger name="org.apache.commons.httpclient" level="WARN" /> + <logger name="org.apache.commons" level="WARN" /> + <logger name="org.apache.coyote" level="WARN" /> + <logger name="org.apache.jasper" level="WARN" /> + + <!-- Camel Related Loggers (including restlet/servlet/jaxrs/cxf logging. + May aid in troubleshooting) --> + <logger name="org.apache.camel" level="WARN" /> + <logger name="org.apache.cxf" level="WARN" /> + <logger name="org.apache.camel.processor.interceptor" level="WARN" /> + <logger name="org.apache.cxf.jaxrs.interceptor" level="WARN" /> + <logger name="org.apache.cxf.service" level="WARN" /> + <logger name="org.restlet" level="DEBUG" /> + <logger name="org.apache.camel.component.restlet" level="DEBUG" /> + + <!-- logback internals logging --> + <logger name="ch.qos.logback.classic" level="INFO" /> + <logger name="ch.qos.logback.core" level="INFO" /> + + <!-- logback jms appenders & loggers definition starts here --> + <!-- logback jms appenders & loggers definition starts here --> + <appender name="auditLogs" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + </filter> + <file>${logDirectory}/Audit.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/Audit.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>"%d [%thread] %-5level %logger{1024} - %msg%n"</pattern> + </encoder> + </appender> + <appender name="perfLogs" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + </filter> + <file>${logDirectory}/Perform.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy"> + <fileNamePattern>${logDirectory}/Perform.%i.log.zip + </fileNamePattern> + <minIndex>1</minIndex> + <maxIndex>9</maxIndex> + </rollingPolicy> + <triggeringPolicy + class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy"> + <maxFileSize>5MB</maxFileSize> + </triggeringPolicy> + <encoder> + <pattern>"%d [%thread] %-5level %logger{1024} - %msg%n"</pattern> + </encoder> + </appender> + <logger name="AuditRecord" level="INFO" additivity="FALSE"> + <appender-ref ref="auditLogs" /> + </logger> + <logger name="AuditRecord_DirectCall" level="INFO" additivity="FALSE"> + <appender-ref ref="auditLogs" /> + </logger> + <logger name="PerfTrackerRecord" level="INFO" additivity="FALSE"> + <appender-ref ref="perfLogs" /> + </logger> + <!-- logback jms appenders & loggers definition ends here --> + + <root level="WARN"> + <appender-ref ref="DEBUG" /> + <appender-ref ref="STDOUT" /> + </root> + +</configuration> diff --git a/kubernetes/clamp/templates/configmap.yaml b/kubernetes/clamp/templates/configmap.yaml index 7a66c64755..e0ed211717 100644 --- a/kubernetes/clamp/templates/configmap.yaml +++ b/kubernetes/clamp/templates/configmap.yaml @@ -25,3 +25,11 @@ metadata: data: {{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }} spring_application_json: {{ tpl .Values.config.springApplicationJson . | quote }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-clamp-filebeat-configmap + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/log/filebeat/*").AsConfig . | indent 2 }} diff --git a/kubernetes/clamp/templates/deployment.yaml b/kubernetes/clamp/templates/deployment.yaml index 38eabeb968..fb06d834f4 100644 --- a/kubernetes/clamp/templates/deployment.yaml +++ b/kubernetes/clamp/templates/deployment.yaml @@ -46,9 +46,24 @@ spec: imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} name: {{ include "common.name" . }}-readiness containers: + # side car containers + - name: {{ include "common.name" . }}-filebeat-onap + image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + volumeMounts: + - name: {{ include "common.fullname" . }}-filebeat-conf + mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + - name: {{ include "common.fullname" . }}-data-filebeat + mountPath: /usr/share/filebeat/datag + - name: {{ include "common.fullname" . }}-logs + mountPath: /var/log/onap - name: {{ include "common.name" . }} image: "{{ include "common.repository" . }}/{{ .Values.image }}" imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + args: + - "-Dcom.att.eelf.logging.file=file:/opt/clamp/logback.xml" + - "" ports: - containerPort: {{ .Values.service.internalPort }} # disable liveness probe when breakpoints set in debugger @@ -66,9 +81,14 @@ spec: initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} volumeMounts: + - name: {{ include "common.fullname" . }}-logs + mountPath: /var/log/onap - mountPath: /opt/clamp/sdc-controllers-config.json name: {{ include "common.fullname" . }}-config subPath: sdc-controllers-config.json + - mountPath: /opt/clamp/logback.xml + name: {{ include "common.fullname" . }}-config + subPath: logback.xml env: - name: SPRING_APPLICATION_JSON valueFrom: @@ -92,5 +112,14 @@ spec: items: - key: sdc-controllers-config.json path: sdc-controllers-config.json + - key: logback.xml + path: logback.xml + - name: {{ include "common.fullname" . }}-filebeat-conf + configMap: + name: {{ .Release.Name }}-clamp-filebeat-configmap + - name: {{ include "common.fullname" . }}-data-filebeat + emptyDir: {} + - name: {{ include "common.fullname" . }}-logs + emptyDir: {} imagePullSecrets: - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/kubernetes/clamp/values.yaml b/kubernetes/clamp/values.yaml index 71752fcfc6..180e3f5983 100644 --- a/kubernetes/clamp/values.yaml +++ b/kubernetes/clamp/values.yaml @@ -37,6 +37,9 @@ debugEnabled: false # Application configuration defaults. ################################################################# config: + log: + logstashServiceName: log-ls + logstashPort: 5044 mysqlPassword: strong_pitchou dataRootDir: /dockerdata-nfs springApplicationJson: > |