diff options
Diffstat (limited to 'kubernetes/log')
10 files changed, 130 insertions, 71 deletions
diff --git a/kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml b/kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml index e7933b8570..abdab8beb5 100644 --- a/kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml +++ b/kubernetes/log/charts/log-elasticsearch/resources/config/elasticsearch.yml @@ -116,7 +116,7 @@ http.port: {{.Values.service.externalPort}} # If a range is specified, the node will bind to the first available port in the range. # Defaults to 9300-9400. # More info: -transport.tcp.port: {{.Values.service.externalPortTcp}} +transport.tcp.port: {{.Values.service.externalPort2}} xpack.graph.enabled: false #Set to false to disable X-Pack graph features. diff --git a/kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml b/kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml index 13caa7e866..fdfc6140fd 100644 --- a/kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml +++ b/kubernetes/log/charts/log-elasticsearch/templates/deployment.yaml @@ -61,8 +61,8 @@ spec: ports: - containerPort: {{ .Values.service.internalPort }} name: {{ .Values.service.name }} - - containerPort: {{ .Values.service.internalPortTcp }} - name: {{ .Values.service.name }}-tcp + - containerPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} # disable liveness probe when breakpoints set in debugger # so K8s doesn't restart unresponsive container {{- if eq .Values.liveness.enabled true }} @@ -74,7 +74,7 @@ spec: {{ end -}} readinessProbe: tcpSocket: - port: {{ .Values.service.internalPortTcp }} + port: {{ .Values.service.internalPort2 }} initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} env: diff --git a/kubernetes/log/charts/log-elasticsearch/templates/service.yaml b/kubernetes/log/charts/log-elasticsearch/templates/service.yaml index 72dfc7e817..cdc2ea8e77 100644 --- a/kubernetes/log/charts/log-elasticsearch/templates/service.yaml +++ b/kubernetes/log/charts/log-elasticsearch/templates/service.yaml @@ -27,19 +27,40 @@ spec: type: {{ .Values.service.type }} ports: {{if eq .Values.service.type "NodePort" -}} - - port: {{ .Values.service.externalPort }} + - port: {{ .Values.service.internalPort }} nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} name: {{ .Values.service.name }} - - port: {{ .Values.service.internalPortTcp }} - nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.externalPortTcp }} - name: {{ .Values.service.name }}-tcp {{- else -}} - port: {{ .Values.service.externalPort }} targetPort: {{ .Values.service.internalPort }} name: {{ .Values.service.name }} - - port: {{ .Values.service.externalPortTcp }} - targetPort: {{ .Values.service.internalPortTcp }} - name: {{ .Values.service.name }}-tcp + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name2 }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type2 }} + ports: + {{if eq .Values.service.type2 "NodePort" -}} + - port: {{ .Values.service.internalPort2 }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} + name: {{ .Values.service.name2 }} + {{- else -}} + - port: {{ .Values.service.externalPort2 }} + targetPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} {{- end}} selector: app: {{ include "common.name" . }} diff --git a/kubernetes/log/charts/log-elasticsearch/values.yaml b/kubernetes/log/charts/log-elasticsearch/values.yaml index 2e6e08509e..d5ac1d1781 100644 --- a/kubernetes/log/charts/log-elasticsearch/values.yaml +++ b/kubernetes/log/charts/log-elasticsearch/values.yaml @@ -82,15 +82,16 @@ persistence: mountSubPathLogs: log service: - #Example service definition with external, internal and node ports. - #Services may use any combination of ports depending on the 'type' of - #service being defined. - type: ClusterIP + type: NodePort name: log-es externalPort: 9200 internalPort: 9200 - externalPortTcp: 9300 - internalPortTcp: 9300 + nodePort: 54 + type2: ClusterIP + name2: log-es-tcp + externalPort2: 9300 + internalPort2: 9300 + ingress: enabled: false diff --git a/kubernetes/log/charts/log-kibana/templates/deployment.yaml b/kubernetes/log/charts/log-kibana/templates/deployment.yaml index bdd5a447be..09dc88fb20 100644 --- a/kubernetes/log/charts/log-kibana/templates/deployment.yaml +++ b/kubernetes/log/charts/log-kibana/templates/deployment.yaml @@ -53,27 +53,30 @@ spec: - containerPort: {{ .Values.service.internalPort }} name: {{ .Values.service.name }} readinessProbe: - tcpSocket: + httpGet: + path: "/" port: {{ .Values.service.internalPort }} initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.readiness.periodSeconds }} + timeoutSeconds: {{ .Values.readiness.timeoutSeconds }} # disable liveness probe when breakpoints set in debugger # so K8s doesn't restart unresponsive container {{- if eq .Values.liveness.enabled true }} livenessProbe: - tcpSocket: + httpGet: + path: "/" port: {{ .Values.service.internalPort }} initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.liveness.periodSeconds }} + timeoutSeconds: {{ .Values.liveness.timeoutSeconds }} {{ end -}} env: volumeMounts: - mountPath: /etc/localtime name: localtime readOnly: true - - mountPath: /usr/share/kibana/config/kibana.yml + - mountPath: /usr/share/kibana/config/ name: {{ include "common.fullname" . }} - subPath: kibana.yml resources: {{ toYaml .Values.resources | indent 12 }} {{- if .Values.nodeSelector }} diff --git a/kubernetes/log/charts/log-kibana/values.yaml b/kubernetes/log/charts/log-kibana/values.yaml index 50c6e52ee0..828e23f70e 100644 --- a/kubernetes/log/charts/log-kibana/values.yaml +++ b/kubernetes/log/charts/log-kibana/values.yaml @@ -19,7 +19,7 @@ global: nodePortPrefix: 302 repositorySecret: eyJuZXh1czMub25hcC5vcmc6MTAwMDEiOnsidXNlcm5hbWUiOiJkb2NrZXIiLCJwYXNzd29yZCI6ImRvY2tlciIsImVtYWlsIjoiQCIsImF1dGgiOiJaRzlqYTJWeU9tUnZZMnRsY2c9PSJ9fQ== readinessRepository: oomk8s - readinessImage: readiness-check:1.1.0 + readinessImage: readiness-check:2.0.0 persistence: {} ################################################################# @@ -52,15 +52,17 @@ affinity: {} # probe configuration parameters liveness: - initialDelaySeconds: 120 + initialDelaySeconds: 180 periodSeconds: 10 + timeoutSeconds: 1 # necessary to disable liveness probe when setting breakpoints # in debugger so K8s doesn't restart unresponsive container enabled: true readiness: - initialDelaySeconds: 10 + initialDelaySeconds: 180 periodSeconds: 10 + timeoutSeconds: 1 service: #Example service definition with external, internal and node ports. diff --git a/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf b/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf index 3b4fd768c3..e46d2bac14 100644 --- a/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf +++ b/kubernetes/log/charts/log-logstash/resources/config/onap-pipeline.conf @@ -37,8 +37,20 @@ input { filter { + grok { + break_on_match => false + match => { + "source" => ["/var/log/onap/(?<componentName>[^/]+)/", + "/var/log/onap/%{GREEDYDATA:componentLogFile}" + ] + } + } + # Filter for log4j xml events if "</log4j:event>" in [message] { + + #mutate { add_field => { "orgmsg_log4j" => "%{message}" } } # Copy of orginal msg for debug + #Filter to parse xml event and retrieve data xml { source => "message" @@ -118,7 +130,7 @@ filter { # Filter for logback events else { -# mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug + #mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug mutate { gsub => [ @@ -128,50 +140,44 @@ filter { 'message', '\t$', '\t' ] } + # The grok below parses the message field for all current logback patterns used by oom components. + # Example logback pattern: %d{"yyyy-MM-dd'T'HH:mm:ss.SSSXXX", UTC}|%X{RequestId}|%msg + # Example grok pattern: %{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message} + # Use the following command to find all logback patterns in oom directory: find oom -name "logback*xml" -exec grep "property.*attern.*value" {} \;|sort|uniq grok { - break_on_match => false match => { - "message" => ["%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}", - "(?<MDCs>.*\t)" - ] - "source" => ["/var/log/onap/(?<componentName>[^/]+)/", - "/var/log/onap/%{GREEDYDATA:componentLogFile}" - ] + "message" => [ + "%{TIMESTAMP_ISO8601:Timestamp}\\t[%{GREEDYDATA:Thread}]\\t%{GREEDYDATA:loglevel}\\t%{JAVACLASS:Logger}\\t%{GREEDYDATA:MDCs}\\t%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:TargetVirtualEntity}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Timer}\|\[%{GREEDYDATA:caller}\]\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:ErrorCode}\|%{GREEDYDATA:ErrorDesc}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ClassName}\|%{GREEDYDATA:message}", + "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}", + "\[%{TIMESTAMP_ISO8601:Timestamp}\|%{LOGLEVEL:loglevel}\|%{GREEDYDATA:Logger}\|%{GREEDYDATA:Thread}\] %{GREEDYDATA:message}" + ] } overwrite => ["message"] } + # The MDCs are key value pairs that are seperated by "," or "\t". Extra space characters are trimmed from the keys and values. kv { source => "MDCs" - field_split => "\t" + field_split => ",\t" trim_key => "\s" trim_value => "\s" remove_field => [ "MDCs" ] } + if (![Timestamp] and [EndTimestamp]) { + mutate { add_field => { "Timestamp" => "%{EndTimestamp}" } } + } date { match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ] target => "Timestamp" } - if [source] == "/var/log/onap/aai/aai-ml/metrics.log" { - csv { - source => "message" - separator => "|" - quote_char => "`" - columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Log level", "Unknown8", "Unknown9", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18", "message"] - } - } - else if [source] == "/var/log/onap/aai/aai-ml/audit.log" { - csv { - source => "message" - separator => "|" - quote_char => "`" - columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Log level", "Unknown6", "Unknown7", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "message"] - } - } - mutate { - remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8", "Unknown9", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18"] + remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8"] } if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") { diff --git a/kubernetes/log/charts/log-logstash/templates/deployment.yaml b/kubernetes/log/charts/log-logstash/templates/deployment.yaml index 682c448da8..7735c82616 100644 --- a/kubernetes/log/charts/log-logstash/templates/deployment.yaml +++ b/kubernetes/log/charts/log-logstash/templates/deployment.yaml @@ -52,8 +52,8 @@ spec: ports: - containerPort: {{ .Values.service.internalPort }} name: {{ .Values.service.name }} - - containerPort: {{ .Values.service.internalPortHttp }} - name: {{ .Values.service.name }}-http + - containerPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} readinessProbe: tcpSocket: port: {{ .Values.service.internalPort }} @@ -73,12 +73,10 @@ spec: - mountPath: /etc/localtime name: localtime readOnly: true - - mountPath: /usr/share/logstash/config/logstash.yml - name: {{ include "common.fullname" . }} - subPath: logstash.yml - - mountPath: /usr/share/logstash/pipeline/onap-pipeline.conf - name: {{ include "common.fullname" . }} - subPath: onap-pipeline.conf + - mountPath: /usr/share/logstash/config/ + name: {{ include "common.fullname" . }}-config + - mountPath: /usr/share/logstash/pipeline/ + name: {{ include "common.fullname" . }}-pipeline resources: {{ toYaml .Values.resources | indent 12 }} {{- if .Values.nodeSelector }} @@ -93,12 +91,16 @@ spec: - name: localtime hostPath: path: /etc/localtime - - name: {{ include "common.fullname" . }} + - name: {{ include "common.fullname" . }}-config configMap: name: {{ include "common.fullname" . }} items: - key: logstash.yml path: logstash.yml + - name: {{ include "common.fullname" . }}-pipeline + configMap: + name: {{ include "common.fullname" . }} + items: - key: onap-pipeline.conf path: onap-pipeline.conf imagePullSecrets: diff --git a/kubernetes/log/charts/log-logstash/templates/service.yaml b/kubernetes/log/charts/log-logstash/templates/service.yaml index ca71aa2224..cdc2ea8e77 100644 --- a/kubernetes/log/charts/log-logstash/templates/service.yaml +++ b/kubernetes/log/charts/log-logstash/templates/service.yaml @@ -27,19 +27,40 @@ spec: type: {{ .Values.service.type }} ports: {{if eq .Values.service.type "NodePort" -}} - - port: {{ .Values.service.externalPort }} + - port: {{ .Values.service.internalPort }} nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort }} name: {{ .Values.service.name }} - - port: {{ .Values.service.internalPortHttp }} - nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.externalPortHttp }} - name: {{ .Values.service.name }}-http {{- else -}} - port: {{ .Values.service.externalPort }} targetPort: {{ .Values.service.internalPort }} name: {{ .Values.service.name }} - - port: {{ .Values.service.externalPortHttp }} - targetPort: {{ .Values.service.internalPortHttp }} - name: {{ .Values.service.name }}-http + {{- end}} + selector: + app: {{ include "common.name" . }} + release: {{ .Release.Name }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.service.name2 }} + namespace: {{ include "common.namespace" . }} + labels: + app: {{ include "common.name" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: +spec: + type: {{ .Values.service.type2 }} + ports: + {{if eq .Values.service.type2 "NodePort" -}} + - port: {{ .Values.service.internalPort2 }} + nodePort: {{ .Values.global.nodePortPrefix | default .Values.nodePortPrefix }}{{ .Values.service.nodePort2 }} + name: {{ .Values.service.name2 }} + {{- else -}} + - port: {{ .Values.service.externalPort2 }} + targetPort: {{ .Values.service.internalPort2 }} + name: {{ .Values.service.name2 }} {{- end}} selector: app: {{ include "common.name" . }} diff --git a/kubernetes/log/charts/log-logstash/values.yaml b/kubernetes/log/charts/log-logstash/values.yaml index bfe50b54a8..207182131b 100644 --- a/kubernetes/log/charts/log-logstash/values.yaml +++ b/kubernetes/log/charts/log-logstash/values.yaml @@ -19,7 +19,7 @@ global: nodePortPrefix: 302 repositorySecret: eyJuZXh1czMub25hcC5vcmc6MTAwMDEiOnsidXNlcm5hbWUiOiJkb2NrZXIiLCJwYXNzd29yZCI6ImRvY2tlciIsImVtYWlsIjoiQCIsImF1dGgiOiJaRzlqYTJWeU9tUnZZMnRsY2c9PSJ9fQ== readinessRepository: oomk8s - readinessImage: readiness-check:1.1.0 + readinessImage: readiness-check:2.0.0 persistence: {} ################################################################# @@ -62,12 +62,15 @@ service: #Example service definition with external, internal and node ports. #Services may use any combination of ports depending on the 'type' of #service being defined. - type: ClusterIP + type: NodePort name: log-ls externalPort: 5044 internalPort: 5044 - externalPortHttp: 9600 - internalPortHttp: 9600 + nodePort: 55 + type2: ClusterIP + name2: log-ls-http + externalPort2: 9600 + internalPort2: 9600 ingress: enabled: false |