diff options
Diffstat (limited to 'kubernetes/log/resources/logstash')
| -rw-r--r-- | kubernetes/log/resources/logstash/conf/logstash.yml | 16 | ||||
| -rw-r--r-- | kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf | 204 |
2 files changed, 220 insertions, 0 deletions
diff --git a/kubernetes/log/resources/logstash/conf/logstash.yml b/kubernetes/log/resources/logstash/conf/logstash.yml new file mode 100644 index 0000000000..f658006503 --- /dev/null +++ b/kubernetes/log/resources/logstash/conf/logstash.yml @@ -0,0 +1,16 @@ +http.host: "0.0.0.0" +## Path where pipeline configurations reside +path.config: /usr/share/logstash/pipeline + +## Type of queue : memeory based or file based +#queue.type: persisted +## Size of queue +#queue.max_bytes: 1024mb +## Setting true makes logstash check periodically for change in pipeline configurations +config.reload.automatic: true + +## xpack configurations +#xpack.monitoring.elasticsearch.url: ["http://10.247.186.12:9200", "http://10.247.186.13:9200"] +#xpack.monitoring.elasticsearch.username: elastic +#xpack.monitoring.elasticsearch.password: changeme +xpack.monitoring.enabled: false diff --git a/kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf b/kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf new file mode 100644 index 0000000000..1a46375047 --- /dev/null +++ b/kubernetes/log/resources/logstash/pipeline/onap-pipeline.conf @@ -0,0 +1,204 @@ +input { + beats { + + ## Add a id to plugin configuration. Can be anything unique. + id => 'beats_plugin' + + ######## Connection configurations ######## + + ## The port to listen on. + port => 5044 + + ## Close Idle clients after the specified time in seconds. Default is 60 seconds + #client_inactivity_timeout => 60 + + ######## Security configurations ######## + + ## Enable encryption. Default false. + #ssl => $filebeat_ssl + + ## ssl certificate path. + #ssl_certificate => $filebeat_ssl_certificate + + ## SSL key to use. + #ssl_key => $filebeat_ssl_key + + ##SSL key passphrase to use. + #ssl_key_passphrase => $filebeat_ssl_key_passphrase + + ## Value can be any of: none, peer, force_peer. + #ssl_verify_mode => $filebeat_ssl_verify_mode + + ## Time in milliseconds for an incomplete ssl handshake to timeout. Default is 10000 ms. + #ssl_handshake_timeout => 10000 + include_codec_tag => false + } +} + + +filter { + # Filter for log4j xml events + if "</log4j:event>" in [message] { + #Filter to parse xml event and retrieve data + xml { + source => "message" + store_xml => false + remove_namespaces => true + target => "xml_content" + xpath => [ "/event/message/text()", "logmsg" , + "/event/@logger", "Logger", + "/event/@timestamp", "Timestamp", + "/event/@level", "loglevel", + "/event/@thread", "Thread", + "/event/throwable/text()", "Exceptionthrowable", + "/event/NDC/text()", "NDCs", + "/event/properties/data/@name","mdcname", + "/event/properties/data/@value","mdcvalue"] + + } + + #Ruby filter to iterate and separate MDCs into documents + ruby { + code => ' + $i = 0 + $num = 0 + if event.get("[mdcname]") + $num = event.get("[mdcname]").length + end + if $num != 0 + until $i > $num do + if event.get("[mdcname]").at($i) and event.get("[mdcvalue]").at($i) + event.set(event.get("[mdcname]").at($i), event.get("[mdcvalue]").at($i)) + end + $i=$i+1 + end + end + ' + } + + #Validations + if [Exceptionthrowable] + { + mutate { + replace => { + "exceptionmessage" => "%{[Exceptionthrowable]}" + } + } + } + + if [NDCs] + { + mutate { + replace => { + "NDC" => "%{[NDCs]}" + } + } + } + + mutate { + replace => { + "Logger" =>"%{[Logger]}" + "logmsg" =>"%{[logmsg]}" + "Timestamp" =>"%{[Timestamp]}" + "loglevel" =>"%{[loglevel]}" + "message" => "%{logmsg}" + "Thread" => "%{[Thread]}" + } + remove_field => ["mdcname", "mdcvalue", "logmsg","Exceptionthrowable","NDCs"] + } + + if [Timestamp] + { + date { + match => ["Timestamp", "UNIX_MS"] + target => "Timestamp" + } + } + } + # Filter for logback events + else { + mutate { + gsub => [ + 'message', '= ', '=null', + 'message', '=\t', '=null ', #This null is followed by a tab + 'message', '\t$', '\t' + ] + } + kv { + field_split => "\t" + trim_key => "\s" + trim_value => "\s" + } + grok { + break_on_match => false + match => { + "message" => "%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}" + } + overwrite => ["message"] + } + } +} + + +output { + elasticsearch { + id => 'onap_es' + + ######### Security configurations ######### + + user => "elastic" + password => "changeme" + + ## The .cer or .pem file to validate the server’s certificate + #cacert => $es_cacert + + ## The keystore used to present a certificate to the server. It can be either .jks or .p12 + #keystore => $es_keystore + #keystore_password => $es_keystore_password + + ## Enable SSL/TLS secured communication to Elasticsearch cluster. + ## Default is not set which in that case depends on the protocol specidfied in hosts list + #ssl => $es_ssl + + ## Option to validate the server’s certificate. Default is true + #ssl_certificate_verification => $es_ssl_certificate_verification + + ## The JKS truststore to validate the server’s certificate. + #truststore => $es_truststore + #truststore_password => $es_truststore_password + + + ######### Elasticsearchcluster and host configurations ######### + +#can specify one or a list of hosts. If sniffing is set, one is enough and others will be auto-discovered +##Also protocol can be specified like ["http://10.247.186.12:9200"] + hosts => ["http://elasticsearch.onap-log:9200"] + + + ## This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. Default is false. + sniffing => true + + ## How long to wait, in seconds, between sniffing attempts. Default is 5 seconds. + #sniffing_delay => 5 + + ## Set the address of a forward HTTP proxy. + #proxy => $es_proxy + + ##Use this if you must run Elasticsearch behind a proxy that remaps the root path for the Elasticsearch HTTP API lives + #path => $es_path + + ######### Elasticsearch request configurations ######### + + ## This setting defines the maximum sized bulk request Logstash will make. + #flush_size => ? + + ######### Document configurations ######### + + index => "onaplogs-%{+YYYY.MM.dd}" + document_type => "logs" + + ## This can be used to associate child documents with a parent using the parent ID. + #parent => "abcd' + } +} + |