Enhance logstash filter for logback

The existing filter was failing for a scenario. When a 'equals to' sign
use to appear in the log message the filter would think of it as a key
value MDC and add a invalid field in the output json. This fix will make
logstash search for MDCs only till before the log message.

Change-Id: I549bde8ec0ea318be37a1b36a1b05adedb55a7cd
Issue-Id: LOG-34
Signed-off-by: pranavdi <Pranav.Dixit@amdocs.com>

1 files changed, 10 insertions, 6 deletions

diff --git a/elasticstack/logstash/conf/onap-pipeline.conf b/elasticstack/logstash/conf/onap-pipeline.conf
index a18b216..5b32038 100644
--- a/elasticstack/logstash/conf/onap-pipeline.conf
+++ b/elasticstack/logstash/conf/onap-pipeline.conf
@@ -124,18 +124,22 @@ filter {
       'message', '\t$', '\t'
     ]
   }
-  kv {
-    field_split => "\t"
-    trim_key => "\s"
-    trim_value => "\s"
-  }
   grok {
     break_on_match => false
     match => {
-      "message" => "%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}"
+      "message" => ["%{TIMESTAMP_ISO8601:Timestamp}\t%{GREEDYDATA:Thread}\t%{SPACE}%{LOGLEVEL:loglevel}%{SPACE}\t%{JAVACLASS:Logger}\t(?:[^\t]+\t)*%{GREEDYDATA:message}",
+                    "(?<MDCs>.*\t)"
+                   ]
     }
     overwrite => ["message"]
   }
+  kv {
+    source => "MDCs"
+    field_split => "\t"
+    trim_key => "\s"
+    trim_value => "\s"
+    remove_field => [ "MDCs" ]
+  }
  }
 }