aboutsummaryrefslogtreecommitdiffstats
path: root/src/main/docker
diff options
context:
space:
mode:
authorosgn422w <gervais-martial.ngueko@intl.att.com>2018-10-30 17:04:03 +0100
committerosgn422w <gervais-martial.ngueko@intl.att.com>2018-10-30 17:27:34 +0100
commite69ed980a37c4f449e01bf0a15635da58bca7512 (patch)
treee5f1252b0df61d7730a8ae8eb3d58582f93c9876 /src/main/docker
parent447d81c985e893c5a5b6b5305cb4e7975717b75b (diff)
skip processing for log
skip specific http processing in case of logfile events Issue-ID: CLAMP-240 Change-Id: I26d38ea99d46b186fa81fda01b63ab207c881057 Signed-off-by: osgn422w <gervais-martial.ngueko@intl.att.com>
Diffstat (limited to 'src/main/docker')
-rw-r--r--src/main/docker/logstash/pipeline/logstash.conf88
1 files changed, 47 insertions, 41 deletions
diff --git a/src/main/docker/logstash/pipeline/logstash.conf b/src/main/docker/logstash/pipeline/logstash.conf
index e6cee9c19..6fe9d9691 100644
--- a/src/main/docker/logstash/pipeline/logstash.conf
+++ b/src/main/docker/logstash/pipeline/logstash.conf
@@ -61,48 +61,54 @@ input {
}
filter {
- # avoid noise if no entry in the list
- if [message] == "[]" {
- drop { }
- }
+ if [type] != "dmaap_log" {
+ #only execute this section for dmaap events from http request
+ #it doesn't apply to dmaap events from log file
- if [http_request_failure] or [@metadata][code] != "200" {
- mutate {
- add_tag => [ "error" ]
- }
- }
+ # avoid noise if no entry in the list
+ if [message] == "[]" {
+ drop { }
+ }
- if "dmaap_source" in [tags] {
- #
- # Dmaap provides a json list, whose items are Strings containing the event
- # provided to Dmaap, which itself is an escaped json.
- #
- # We first need to parse the json as we have to use the plaintext as it cannot
- # work with list of events, then split that list into multiple string events,
- # that we then transform into json.
- #
- json {
- source => "[message]"
- target => "message"
- }
- ruby {
- code => "
- for ev in event.get('message', [])
- ev.set('@metadata', event.get('@metadata'))
- end
- "
- }
-
- split {
- field => "message"
- }
- json {
- source => "message"
- }
- mutate {
- remove_field => [ "message" ]
- }
- }
+ if [http_request_failure] or [@metadata][code] != "200" {
+ mutate {
+ add_tag => [ "error" ]
+ }
+ }
+
+ if "dmaap_source" in [tags] {
+ #
+ # Dmaap provides a json list, whose items are Strings containing the event
+ # provided to Dmaap, which itself is an escaped json.
+ #
+ # We first need to parse the json as we have to use the plaintext as it cannot
+ # work with list of events, then split that list into multiple string events,
+ # that we then transform into json.
+ #
+ json {
+ source => "[message]"
+ target => "message"
+ }
+ ruby {
+ code => "
+ for ev in event.get('message', [])
+ ev.set('@metadata', event.get('@metadata'))
+ end
+ "
+ }
+
+ split {
+ field => "message"
+ }
+ json {
+ source => "message"
+ }
+ mutate {
+ remove_field => [ "message" ]
+ }
+ }
+ }
+ #now start the common, to both http request and log file events, processing
#
# Some timestamps are expressed as milliseconds, some are in microseconds
@@ -250,7 +256,7 @@ output {
elasticsearch {
codec => "json"
hosts => ["${elasticsearch_base_url}"]
- index => "events-%{+YYYY.MM.DD}" # creates daily indexes
+ index => "events-raw-%{+YYYY.MM.DD}" # creates daily indexes
doc_as_upsert => true
}
}