diff options
| -rw-r--r-- | elasticstack/logstash/conf/onap-pipeline.conf | 65 |
1 files changed, 57 insertions, 8 deletions
diff --git a/elasticstack/logstash/conf/onap-pipeline.conf b/elasticstack/logstash/conf/onap-pipeline.conf index a382edc..ed3240d 100644 --- a/elasticstack/logstash/conf/onap-pipeline.conf +++ b/elasticstack/logstash/conf/onap-pipeline.conf @@ -117,8 +117,12 @@ filter { } # Filter for logback events else { + +# mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug + mutate { gsub => [ + 'message', ' = ', '=', 'message', '= ', '=null', 'message', '=\t', '=null ', #This null is followed by a tab 'message', '\t$', '\t' @@ -130,7 +134,9 @@ filter { "message" => ["%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}", "(?<MDCs>.*\t)" ] - "source" => ["/var/log/onap/(?<componentName>[^/]+)/"] + "source" => ["/var/log/onap/(?<componentName>[^/]+)/", + "/var/log/onap/%{GREEDYDATA:componentLogFile}" + ] } overwrite => ["message"] } @@ -141,8 +147,50 @@ filter { trim_value => "\s" remove_field => [ "MDCs" ] } - } -} + + date { + match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ] + target => "Timestamp" + } + + if [source] == "/var/log/onap/aai/aai-ml/metrics.log" { + csv { + source => "message" + separator => "|" + quote_char => "`" + columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Log level", "Unknown8", "Unknown9", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18", "message"] + } + } + else if [source] == "/var/log/onap/aai/aai-ml/audit.log" { + csv { + source => "message" + separator => "|" + quote_char => "`" + columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Log level", "Unknown6", "Unknown7", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "message"] + } + } + + mutate { + remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8", "Unknown9", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18"] + } + + if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") { + + #Parse kvps in message + kv { + field_split => "\s" + trim_key => "\s" + trim_value => "\s" + } + + #If Request Id is missing and DID is present use as RequestId + if (![RequestId] and [DID] =~ /.+/) { + mutate { add_field => { "RequestId" => "%{DID}" } } + } + } + + } #Close else statement for logback events +} #Close filter output { @@ -154,7 +202,7 @@ output { user => $es_user password => $es_password - ## The .cer or .pem file to validate the server’s certificate + ## The .cer or .pem file to validate the server's certificate #cacert => $es_cacert ## The keystore used to present a certificate to the server. It can be either .jks or .p12 @@ -165,10 +213,10 @@ output { ## Default is not set which in that case depends on the protocol specidfied in hosts list #ssl => $es_ssl - ## Option to validate the server’s certificate. Default is true + ## Option to validate the server's certificate. Default is true #ssl_certificate_verification => $es_ssl_certificate_verification - ## The JKS truststore to validate the server’s certificate. + ## The JKS truststore to validate the server's certificate. #truststore => $es_truststore #truststore_password => $es_truststore_password @@ -177,7 +225,7 @@ output { #can specify one or a list of hosts. If sniffing is set, one is enough and others will be auto-discovered ##Also protocol can be specified like ["http://10.247.186.12:9200"] - hosts => ["http://elasticsearch.onap:9200"] + hosts => ["http://elasticsearch.onap-log:9200"] ## This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. Default is false. @@ -205,4 +253,5 @@ output { ## This can be used to associate child documents with a parent using the parent ID. #parent => "abcd' } -}
\ No newline at end of file +} + |