diff options
author | Michael O'Brien <frank.obrien@amdocs.com> | 2018-07-23 10:07:49 -0400 |
---|---|---|
committer | Michael O'Brien <frank.obrien@amdocs.com> | 2018-07-30 17:05:31 -0400 |
commit | 396402cdd65c3f1a5b1fe2ab7fc0dd62f9dc35bd (patch) | |
tree | df15f46731eca7299f4be2204272c1c2600e5177 /reference/logging-kubernetes/logdemonode/charts | |
parent | 268e848b1494dc09d614d8479f70bf9aefa69166 (diff) |
add filebeat to k8s log demo
Change-Id: I1c229a9e5f7c9379d8d9f9fdf724908f083925d9
Issue-ID: LOG-137
Signed-off-by: Michael O'Brien <frank.obrien@amdocs.com>
Diffstat (limited to 'reference/logging-kubernetes/logdemonode/charts')
8 files changed, 496 insertions, 0 deletions
diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/Chart.yaml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/Chart.yaml new file mode 100644 index 0000000..a8cbcf9 --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/Chart.yaml @@ -0,0 +1,18 @@ +# Copyright © 2018 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +description: Logging reference demo node container - use with demo master +name: logdemonode +version: 2.0.0 diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/requirements.yaml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/requirements.yaml new file mode 100644 index 0000000..97971aa --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/requirements.yaml @@ -0,0 +1,18 @@ +# Copyright © 2018 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dependencies: + - name: common + version: ~2.0.0 + repository: '@local'
\ No newline at end of file diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/resources/config/log/filebeat/filebeat.yml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/resources/config/log/filebeat/filebeat.yml new file mode 100644 index 0000000..b0d4690 --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/resources/config/log/filebeat/filebeat.yml @@ -0,0 +1,41 @@ +filebeat.prospectors: +#it is mandatory, in our case it's log +- input_type: log + #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory. + paths: + - /var/log/onap/*/*/*/*.log + - /var/log/onap/*/*/*.log + - /var/log/onap/*/*.log + #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive + ignore_older: 48h + # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit + clean_inactive: 96h + + +# Name of the registry file. If a relative path is used, it is considered relative to the +# data path. Else full qualified file name. +#filebeat.registry_file: ${path.data}/registry + + +output.logstash: + #List of logstash server ip addresses with port number. + #But, in our case, this will be the loadbalancer IP address. + #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately. + hosts: ["{{.Values.config.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.config.logstashPort}}"] + #If enable will do load balancing among availabe Logstash, automatically. + loadbalance: true + + #The list of root certificates for server verifications. + #If certificate_authorities is empty or not set, the trusted + #certificate authorities of the host system are used. + #ssl.certificate_authorities: $ssl.certificate_authorities + + #The path to the certificate for SSL client authentication. If the certificate is not specified, + #client authentication is not available. + #ssl.certificate: $ssl.certificate + + #The client certificate key used for client authentication. + #ssl.key: $ssl.key + + #The passphrase used to decrypt an encrypted key stored in the configured key file + #ssl.key_passphrase: $ssl.key_passphrase diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/resources/config/logback.xml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/resources/config/logback.xml new file mode 100644 index 0000000..eddb4a8 --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/resources/config/logback.xml @@ -0,0 +1,118 @@ +<configuration scan="false" debug="true"> + <property name="p_tim" value="%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}"/> + <property name="p_lvl" value="%level"/> + <property name="p_log" value="%logger"/> + <property name="p_mdc" value="%replace(%replace(%mdc){'\t','\\\\t'}){'\n', '\\\\n'}"/> + <property name="p_msg" value="%replace(%replace(%msg){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_exc" value="%replace(%replace(%rootException){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_mak" value="%replace(%replace(%marker){'\t', '\\\\t'}){'\n','\\\\n'}"/> + <property name="p_thr" value="%thread"/> + <property name="pattern" value="%nopexception${p_tim}\t${p_thr}\t${p_lvl}\t${p_log}\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n"/> + + <property name="logDir" value="/var/log/onap" /> + <property name="debugDir" value="/var/log/onap" /> + + <property name="componentName" value="logdemonode"></property> + <property name="subComponentName" value="node"></property> + + <property name="errorLogName" value="error" /> + <property name="metricsLogName" value="metrics" /> + <property name="auditLogName" value="audit" /> + <property name="debugLogName" value="debug" /> + + <property name="errorPattern" value="%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}|%X{RequestId}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%.-5level|%X{ErrorCode}|%X{ErrorDesc}|%msg%n\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n" /> + <property name="debugPattern" value="%d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}|%X{RequestId}|%msg%n\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n" /> + + <property name="auditPattern" value="%X{BeginTimestamp}|%X{EndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread||%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{RemoteHost}||||||||%msg%n" /> + <property name="metricPattern" value="%X{BeginTimestamp}|%X{EndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread||%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{RemoteHost}||||%X{TargetVirtualEntity}|||||%msg%n" /> + <property name="logDirectory" value="${logDir}/${componentName}/${subComponentName}" /> + <property name="debugLogDirectory" value="${debugDir}/${componentName}/${subComponentName}" /> + + <appender name="EELFAudit" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${auditLogName}.log</file> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/${auditLogName}.log.%d</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${auditPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFAudit" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFAudit" /> + </appender> + + <appender name="EELFMetrics" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${metricsLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/${metricsLogName}.log.%d</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${metricPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFMetrics" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFMetrics"/> + </appender> + + <appender name="EELFError" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${logDirectory}/${errorLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${logDirectory}/${errorLogName}.log.%d</fileNamePattern> + </rollingPolicy> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + <encoder> + <pattern>${errorPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFError" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFError"/> + </appender> + + <appender name="EELFDebug" + class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>${debugLogDirectory}/${debugLogName}.log</file> + <rollingPolicy + class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>${debugLogDirectory}/${debugLogName}.log.%d</fileNamePattern> + </rollingPolicy> + <encoder> + <pattern>${debugPattern}</pattern> + </encoder> + </appender> + + <appender name="asyncEELFDebug" class="ch.qos.logback.classic.AsyncAppender"> + <queueSize>256</queueSize> + <appender-ref ref="EELFDebug" /> + <includeCallerData>true</includeCallerData> + </appender> + + <logger name="com.att.eelf.audit" level="info" additivity="false"> + <appender-ref ref="asyncEELFAudit" /> + </logger> + + <logger name="com.att.eelf.metrics" level="info" additivity="false"> + <appender-ref ref="asyncEELFMetrics" /> + </logger> + + <logger name="com.att.eelf.error" level="debug" additivity="false"> + <appender-ref ref="asyncEELFError" /> + </logger> + + <root level="INFO"> + <appender-ref ref="asyncEELFDebug" /> + </root> + +</configuration> + diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/configmap.yaml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/configmap.yaml new file mode 100644 index 0000000..3f60531 --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/configmap.yaml @@ -0,0 +1,30 @@ +# Copyright © 2018 Amdocs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-logback-configmap + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/logback.xml").AsConfig . | indent 2 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.fullname" . }}-filebeat-configmap + namespace: {{ include "common.namespace" . }} +data: +{{ tpl (.Files.Glob "resources/config/log/filebeat/filebeat.yml").AsConfig . | indent 2 }} + diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/deployment.yaml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/deployment.yaml new file mode 100644 index 0000000..58ce5f7 --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/deployment.yaml @@ -0,0 +1,118 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ include "common.fullname" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + template: + metadata: + labels: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} + spec: + initContainers: +#Example init container for dependency checking + containers: + # side car container + - name: {{ include "common.name" . }}-filebeat-onap + image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} + volumeMounts: + - name: {{ include "common.fullname" . }}-filebeat-conf + mountPath: /usr/share/filebeat/filebeat.yml + subPath: filebeat.yml + - name: {{ include "common.fullname" . }}-data-filebeat + mountPath: /usr/share/filebeat/data + - name: {{ include "common.fullname" . }}-logs + mountPath: /var/log/onap + + - name: {{ include "common.name" . }} + image: "{{ include "common.repository" . }}/{{ .Values.image }}" + imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }} +# args: +# - "-Dcom.att.eelf.logging.file=file:/opt/logdemonode/logback.xml" +# - "" + ports: + - containerPort: {{ .Values.service.internalPort }} + # disable liveness probe when breakpoints set in debugger + # so K8s doesn't restart unresponsive container + {{ if .Values.liveness.enabled }} + livenessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} + periodSeconds: {{ .Values.liveness.periodSeconds }} + {{ end }} + readinessProbe: + tcpSocket: + port: {{ .Values.service.internalPort }} + initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} + periodSeconds: {{ .Values.readiness.periodSeconds }} + env: +#Example environment variable passed to container +# - name: DEBUG_FLAG +# value: {{ .Values.global.debugEnabled | default .Values.debugEnabled | quote }} + volumeMounts: + - name: {{ include "common.fullname" . }}-logs + mountPath: /var/log/onap + - name: {{ include "common.fullname" . }}-logs-config + mountPath: /opt/logback.xml + subPath: logback.xml + - mountPath: /etc/localtime + name: localtime + readOnly: true +#Example config file mount into container +# - mountPath: /opt/app/application.properties +# name: {{ include "common.name" . }}-config +# subPath: application.properties + resources: +{{ toYaml .Values.resources | indent 12 }} + {{- if .Values.nodeSelector }} + nodeSelector: +{{ toYaml .Values.nodeSelector | indent 10 }} + {{- end -}} + {{- if .Values.affinity }} + affinity: +{{ toYaml .Values.affinity | indent 10 }} + {{- end }} + + volumes: + - name: localtime + hostPath: + path: /etc/localtime +#Example config file mount into container + - name: {{ include "common.fullname" . }}-logs-config + configMap: + name: {{ include "common.fullname" . }}-logback-configmap + items: + - key: logback.xml + path: logback.xml + - name: {{ include "common.fullname" . }}-filebeat-conf + configMap: + name: {{ include "common.fullname" . }}-filebeat-configmap + - name: {{ include "common.fullname" . }}-data-filebeat + emptyDir: {} + - name: {{ include "common.fullname" . }}-logs + emptyDir: {} + imagePullSecrets: + - name: "{{ include "common.namespace" . }}-docker-registry-key" diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/service.yaml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/service.yaml new file mode 100644 index 0000000..e77f3b3 --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/templates/service.yaml @@ -0,0 +1,53 @@ +# Copyright © 2017 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.servicename" . }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: +# Example MSB registration annotation +# msb.onap.org/service-info: '[ +# { +# "serviceName": "so", +# "version": "v1", +# "url": "/ecomp/mso/infra", +# "protocol": "REST" +# "port": "8080", +# "visualRange":"1" +# } +# ]' +spec: + type: {{ .Values.service.type }} + ports: + {{if eq .Values.service.type "NodePort" -}} + - port: {{ .Values.service.externalPort }} + #Example internal target port if required + #targetPort: {{ .Values.service.internalPort }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} + name: {{ .Values.service.portName | default "http" }} + {{- else -}} + - port: {{ .Values.service.externalPort }} + targetPort: {{ .Values.service.internalPort }} + name: {{ .Values.service.portName | default "http" }} + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} diff --git a/reference/logging-kubernetes/logdemonode/charts/logdemonode/values.yaml b/reference/logging-kubernetes/logdemonode/charts/logdemonode/values.yaml new file mode 100644 index 0000000..8772954 --- /dev/null +++ b/reference/logging-kubernetes/logdemonode/charts/logdemonode/values.yaml @@ -0,0 +1,100 @@ +# Copyright © 2018 Amdocs, Bell Canada +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################################# +# Global configuration defaults. +################################################################# +global: + nodePortPrefix: 302 + #repository: nexus3.onap.org:10001 + repository: oomk8s +# readinessRepository: oomk8s +# readinessImage: readiness-check:2.0.0 +# loggingRepository: docker.elastic.co +# loggingImage: beats/filebeat:5.5.0 + +################################################################# +# Application configuration defaults. +################################################################# +# application image +#repository: nexus3.onap.org:10001 +repository: oomk8s +image: logging-demo-nbi:0.0.3 +pullPolicy: Always + +# flag to enable debugging - application support required +debugEnabled: false + +# application configuration +# Example: +config: + logstashServiceName: log-ls + logstashPort: 5044 + +# default number of instances +replicaCount: 1 + +nodeSelector: {} + +affinity: {} + +# probe configuration parameters +liveness: + initialDelaySeconds: 10 + periodSeconds: 10 + # necessary to disable liveness probe when setting breakpoints + # in debugger so K8s doesn't restart unresponsive container + enabled: true + +readiness: + initialDelaySeconds: 10 + periodSeconds: 10 + +#Example service definition with external, internal and node ports. +service: + # The default service name (exposed in the service.yaml) will be the same + # name as the chart. If the service name needs to be overriden (such as + # when a subchart is shared), uncomment the value below. + #name: <service-name-override> + + #Services may use any combination of ports depending on the 'type' of + #service being defined. + type: NodePort + externalPort: 8080 + internalPort: 8080 + nodePort: 58 + # optional port name override - default can be defined in service.yaml + #portName: http + +ingress: + enabled: false + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # + # Example: + # Configure resource requests and limits + # ref: http://kubernetes.io/docs/user-guide/compute-resources/ + # Minimum memory for development is 2 CPU cores and 4GB memory + # Minimum memory for production is 4 CPU cores and 8GB memory +#resources: +# limits: +# cpu: 2 +# memory: 4Gi +# requests: +# cpu: 2 +# memory: 4Gi |