summaryrefslogtreecommitdiffstats
path: root/elasticstack
diff options
context:
space:
mode:
authorshanedaniel <shane.daniel@amdocs.com>2017-10-25 16:11:44 -0400
committershanedaniel <shane.daniel@amdocs.com>2017-10-25 16:11:55 -0400
commit9d948492a9b30438dbeded28d1e699e801230961 (patch)
treeeda83819483221289e51429850e5b14c8b311cd1 /elasticstack
parent3ddb70bbe0413557c4c846addb9f4f9454055001 (diff)
Add logstash parsing for SDC-BE and AAI-ML
Issue-ID: LOG-87 Change-Id: Iedcc36287a3858335a8801852141323acb6f4545 Signed-off-by: shanedaniel <shane.daniel@amdocs.com>
Diffstat (limited to 'elasticstack')
-rw-r--r--elasticstack/logstash/conf/onap-pipeline.conf65
1 files changed, 57 insertions, 8 deletions
diff --git a/elasticstack/logstash/conf/onap-pipeline.conf b/elasticstack/logstash/conf/onap-pipeline.conf
index a382edc..ed3240d 100644
--- a/elasticstack/logstash/conf/onap-pipeline.conf
+++ b/elasticstack/logstash/conf/onap-pipeline.conf
@@ -117,8 +117,12 @@ filter {
}
# Filter for logback events
else {
+
+# mutate { add_field => { "orgmsg" => "%{message}" } } # Copy of orginal msg for debug
+
mutate {
gsub => [
+ 'message', ' = ', '=',
'message', '= ', '=null',
'message', '=\t', '=null ', #This null is followed by a tab
'message', '\t$', '\t'
@@ -130,7 +134,9 @@ filter {
"message" => ["%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}",
"(?<MDCs>.*\t)"
]
- "source" => ["/var/log/onap/(?<componentName>[^/]+)/"]
+ "source" => ["/var/log/onap/(?<componentName>[^/]+)/",
+ "/var/log/onap/%{GREEDYDATA:componentLogFile}"
+ ]
}
overwrite => ["message"]
}
@@ -141,8 +147,50 @@ filter {
trim_value => "\s"
remove_field => [ "MDCs" ]
}
- }
-}
+
+ date {
+ match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ]
+ target => "Timestamp"
+ }
+
+ if [source] == "/var/log/onap/aai/aai-ml/metrics.log" {
+ csv {
+ source => "message"
+ separator => "|"
+ quote_char => "`"
+ columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Log level", "Unknown8", "Unknown9", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18", "message"]
+ }
+ }
+ else if [source] == "/var/log/onap/aai/aai-ml/audit.log" {
+ csv {
+ source => "message"
+ separator => "|"
+ quote_char => "`"
+ columns => ["Begin TS", "End TS", "DuplicateRequestID", "Unknown1", "threadID", "phys/virt server name", "service name", "Partner Name", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Log level", "Unknown6", "Unknown7", "Status code", "Server", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "message"]
+ }
+ }
+
+ mutate {
+ remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8", "Unknown9", "Unknown10", "Unknown11", "Unknown12", "Unknown13", "Unknown14", "Unknown15", "Unknown16", "Unknown17", "Unknown18"]
+ }
+
+ if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") {
+
+ #Parse kvps in message
+ kv {
+ field_split => "\s"
+ trim_key => "\s"
+ trim_value => "\s"
+ }
+
+ #If Request Id is missing and DID is present use as RequestId
+ if (![RequestId] and [DID] =~ /.+/) {
+ mutate { add_field => { "RequestId" => "%{DID}" } }
+ }
+ }
+
+ } #Close else statement for logback events
+} #Close filter
output {
@@ -154,7 +202,7 @@ output {
user => $es_user
password => $es_password
- ## The .cer or .pem file to validate the server’s certificate
+ ## The .cer or .pem file to validate the server's certificate
#cacert => $es_cacert
## The keystore used to present a certificate to the server. It can be either .jks or .p12
@@ -165,10 +213,10 @@ output {
## Default is not set which in that case depends on the protocol specidfied in hosts list
#ssl => $es_ssl
- ## Option to validate the server’s certificate. Default is true
+ ## Option to validate the server's certificate. Default is true
#ssl_certificate_verification => $es_ssl_certificate_verification
- ## The JKS truststore to validate the server’s certificate.
+ ## The JKS truststore to validate the server's certificate.
#truststore => $es_truststore
#truststore_password => $es_truststore_password
@@ -177,7 +225,7 @@ output {
#can specify one or a list of hosts. If sniffing is set, one is enough and others will be auto-discovered
##Also protocol can be specified like ["http://10.247.186.12:9200"]
- hosts => ["http://elasticsearch.onap:9200"]
+ hosts => ["http://elasticsearch.onap-log:9200"]
## This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. Default is false.
@@ -205,4 +253,5 @@ output {
## This can be used to associate child documents with a parent using the parent ID.
#parent => "abcd'
}
-} \ No newline at end of file
+}
+