[LOG] Uses new tpls for repos / images

This commit makes Log chart to use the new generator for repositories and
images.

Issue-ID: OOM-2364
Change-Id: Ie93c5fccc2c9ed6026cd8dd93a3d7e141d1bed83
Signed-off-by: Jakub Latusek <j.latusek@samsung.com>

2 files changed, 308 insertions, 0 deletions

diff --git a/kubernetes/log/components/log-logstash/resources/config/logstash.yml b/kubernetes/log/components/log-logstash/resources/config/logstash.yml
new file mode 100644
index 0000000000..7c3bd8f851
--- /dev/null
+++ b/kubernetes/log/components/log-logstash/resources/config/logstash.yml
@@ -0,0 +1,32 @@
+{{/*
+# Copyright © 2018  AT&T, Amdocs, Bell Canada Intellectual Property.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+http.host: "0.0.0.0"
+pipeline.workers: 3
+## Path where pipeline configurations reside
+path.config: /usr/share/logstash/pipeline
+
+## Type of queue : memeory based or file based
+#queue.type: persisted
+## Size of queue
+#queue.max_bytes: 1024mb
+## Setting true makes logstash check periodically for change in pipeline configurations
+config.reload.automatic: true
+
+## xpack configurations
+#xpack.monitoring.elasticsearch.url: ["http://10.247.186.12:9200", "http://10.247.186.13:9200"]
+#xpack.monitoring.elasticsearch.username: elastic
+#xpack.monitoring.elasticsearch.password: changeme
+xpack.monitoring.enabled: false
diff --git a/kubernetes/log/components/log-logstash/resources/config/onap-pipeline.conf b/kubernetes/log/components/log-logstash/resources/config/onap-pipeline.conf
new file mode 100644
index 0000000000..d6b0696b81
--- /dev/null
+++ b/kubernetes/log/components/log-logstash/resources/config/onap-pipeline.conf
@@ -0,0 +1,276 @@
+{{/*
+# Copyright © 2018  AT&T, Amdocs, Bell Canada Intellectual Property.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+input {
+ beats {
+
+ ## Add a id to plugin configuration. Can be anything unique.
+ id => 'beats_plugin'
+
+ ######## Connection configurations ########
+
+ ## The port to listen on.
+ port => {{.Values.service.externalPort}}
+
+ ## Close Idle clients after the specified time in seconds. Default is 60 seconds
+ #client_inactivity_timeout => 60
+
+ ######## Security configurations ########
+
+ ## Enable encryption. Default false.
+ #ssl => $filebeat_ssl
+
+ ## ssl certificate path.
+ #ssl_certificate => $filebeat_ssl_certificate
+
+ ## SSL key to use.
+ #ssl_key => $filebeat_ssl_key
+
+ ##SSL key passphrase to use.
+ #ssl_key_passphrase => $filebeat_ssl_key_passphrase
+
+ ## Value can be any of: none, peer, force_peer.
+ #ssl_verify_mode => $filebeat_ssl_verify_mode
+
+ ## Time in milliseconds for an incomplete ssl handshake to timeout. Default is 10000 ms.
+ #ssl_handshake_timeout => 10000
+ include_codec_tag => false
+ }
+}
+
+
+filter {
+  grok {
+    break_on_match => false
+    match => {
+      "source" => ["/var/log/onap/(?<componentName>[^/]+)/",
+                   "/var/log/onap/%{GREEDYDATA:componentLogFile}"
+                  ]
+    }
+  }
+
+ # Filter for log4j xml events
+ if "</log4j:event>" in [message] {
+
+   #mutate { add_field => { "orgmsg_log4j" => "%{message}" } }    # Copy of orginal msg for debug
+
+   #Filter to parse xml event and retrieve data
+   xml {
+     source => "message"
+     store_xml => false
+     remove_namespaces => true
+     target => "xml_content"
+     xpath => [ "/event/message/text()", "logmsg" ,
+                "/event/@logger", "Logger",
+                "/event/@timestamp", "Timestamp",
+                "/event/@level", "loglevel",
+                "/event/@thread", "Thread",
+                "/event/throwable/text()", "Exceptionthrowable",
+                "/event/NDC/text()", "NDCs",
+                "/event/properties/data/@name","mdcname",
+                "/event/properties/data/@value","mdcvalue"]
+
+    }
+
+   #Ruby filter to iterate and separate MDCs into documents
+   ruby {
+     code => '
+       $i = 0
+       $num = 0
+       if event.get("[mdcname]")
+         $num = event.get("[mdcname]").length
+       end
+	   if $num != 0
+        until $i > $num do
+         if event.get("[mdcname]").at($i) and event.get("[mdcvalue]").at($i)
+            event.set(event.get("[mdcname]").at($i), event.get("[mdcvalue]").at($i))
+         end
+         $i=$i+1
+        end
+	   end
+          '
+    }
+
+   #Validations
+   if [Exceptionthrowable]
+   {
+      mutate {
+        replace => {
+           "exceptionmessage" => "%{[Exceptionthrowable]}"
+        }
+      }
+    }
+
+   if [NDCs]
+   {
+      mutate {
+        replace => {
+          "NDC" => "%{[NDCs]}"
+        }
+      }
+   }
+
+   mutate {
+     replace => {
+        "Logger" =>"%{[Logger]}"
+        "logmsg" =>"%{[logmsg]}"
+        "Timestamp" =>"%{[Timestamp]}"
+        "loglevel" =>"%{[loglevel]}"
+        "message" => "%{logmsg}"
+        "Thread" => "%{[Thread]}"
+     }
+     remove_field => ["mdcname", "mdcvalue", "logmsg","Exceptionthrowable","NDCs"]
+   }
+
+   if [Timestamp]
+   {
+     date {
+        match => ["Timestamp", "UNIX_MS"]
+        target => "Timestamp"
+     }
+   }
+ }
+ # Filter for logback events
+ else {
+
+  #mutate { add_field => { "orgmsg" => "%{message}" } }    # Copy of orginal msg for debug
+
+  mutate {
+    gsub => [
+      'message', ' = ', '=',
+      'message', '= ', '=null',
+      'message', '=\t', '=null	', #This null is followed by a tab
+      'message', '\t$', '\t'
+    ]
+  }
+  # The grok below parses the message field for all current logback patterns used by oom components.
+  # Example logback pattern: %d{&quot;yyyy-MM-dd'T'HH:mm:ss.SSSXXX&quot;, UTC}|%X{RequestId}|%msg
+  # Example grok pattern:    %{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}
+  # Use the following command to find all logback patterns in oom directory: find oom -name "logback*xml" -exec grep "property.*attern.*value" {} \;|sort|uniq
+  grok {
+    match => {
+      "message" => [
+                    "%{TIMESTAMP_ISO8601:Timestamp}\\t[%{GREEDYDATA:Thread}]\\t%{GREEDYDATA:loglevel}\\t%{JAVACLASS:Logger}\\t%{GREEDYDATA:MDCs}\\t%{GREEDYDATA:message}",
+                    "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:TargetVirtualEntity}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}",
+                    "%{TIMESTAMP_ISO8601:BeginTimestamp}\|%{TIMESTAMP_ISO8601:EndTimestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:Unknown1}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:StatusCode}\|%{GREEDYDATA:ResponseCode}\|%{GREEDYDATA:ResponseDesc}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{GREEDYDATA:Timer}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Unknown2}\|%{GREEDYDATA:Unknown3}\|%{GREEDYDATA:Unknown4}\|%{GREEDYDATA:Unknown5}\|%{GREEDYDATA:Unknown6}\|%{GREEDYDATA:Unknown7}\|%{GREEDYDATA:Unknown8}\|%{GREEDYDATA:message}",
+                    "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:ServiceInstanceId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{UUID:InstanceUUID}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:AlertSeverity}\|%{IP:ServerIPAddress}\|%{HOSTNAME:ServerFQDN}\|%{IPORHOST:RemoteHost}\|%{GREEDYDATA:Timer}\|\[%{GREEDYDATA:caller}\]\|%{GREEDYDATA:message}",
+                    "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ServiceName}\|%{GREEDYDATA:PartnerName}\|%{GREEDYDATA:TargetEntity}\|%{GREEDYDATA:TargetServiceName}\|%{GREEDYDATA:loglevel}\|%{GREEDYDATA:ErrorCode}\|%{GREEDYDATA:ErrorDesc}\|%{GREEDYDATA:message}",
+                    "%{TIMESTAMP_ISO8601:Timestamp}\|%{GREEDYDATA:RequestId}\|%{GREEDYDATA:Thread}\|%{GREEDYDATA:ClassName}\|%{GREEDYDATA:message}",
+                    "%{TIMESTAMP_ISO8601:Timestamp}\|%{UUID:RequestId}\|%{GREEDYDATA:message}",
+                    "\[%{TIMESTAMP_ISO8601:Timestamp}\|%{LOGLEVEL:loglevel}\|%{GREEDYDATA:Logger}\|%{GREEDYDATA:Thread}\] %{GREEDYDATA:message}"
+      ]
+    }
+    overwrite => ["message"]
+  }
+  # The MDCs are key value pairs that are seperated by "," or "\t". Extra space characters are trimmed from the keys and values.
+  kv {
+    source => "MDCs"
+    field_split => ",\t"
+    trim_key => "\s"
+    trim_value => "\s"
+    remove_field => [ "MDCs" ]
+  }
+
+  if (![Timestamp] and [EndTimestamp]) {
+    mutate { add_field => { "Timestamp" => "%{EndTimestamp}" } }
+  }
+  date {
+    match => [ "Timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss,SSS" ]
+    target => "Timestamp"
+  }
+
+  mutate {
+    remove_field => ["DuplicateRequestID", "Unknown1", "Unknown2", "Unknown3", "Unknown4", "Unknown5", "Unknown6", "Unknown7", "Unknown8"]
+  }
+
+  if ([source] == "/var/log/onap/sdc/sdc-be/audit.log") {
+    #Parse kvps in message
+    kv {
+      field_split => "\s"
+      trim_key => "\s"
+      trim_value => "\s"
+    }
+
+    #If Request Id is missing and DID is present use as RequestId
+    if (![RequestId] and [DID] =~ /.+/) {
+      mutate { add_field => { "RequestId" => "%{DID}" } }
+    }
+  }
+
+ } #Close else statement for logback events
+} #Close filter
+
+
+output {
+ elasticsearch {
+ id => 'onap_es'
+
+ ######### Security configurations #########
+
+ user => "elastic"
+ password => "changeme"
+
+ ## The .cer or .pem file to validate the server's certificate
+ #cacert => $es_cacert
+
+ ## The keystore used to present a certificate to the server. It can be either .jks or .p12
+ #keystore => $es_keystore
+ #keystore_password => $es_keystore_password
+
+ ## Enable SSL/TLS secured communication to Elasticsearch cluster.
+ ## Default is not set which in that case depends on the protocol specidfied in hosts list
+ #ssl => $es_ssl
+
+ ## Option to validate the server's certificate. Default is true
+ #ssl_certificate_verification => $es_ssl_certificate_verification
+
+ ## The JKS truststore to validate the server's certificate.
+ #truststore => $es_truststore
+ #truststore_password => $es_truststore_password
+
+
+ ######### Elasticsearchcluster and host configurations #########
+
+ ##can specify one or a list of hosts. If sniffing is set, one is enough and others will be auto-discovered
+ hosts => ["http://{{.Values.config.elasticsearchServiceName}}.{{.Release.Namespace}}:{{.Values.config.elasticsearchPort}}"]
+
+
+ ## This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. Default is false.
+ sniffing => true
+
+ ## How long to wait, in seconds, between sniffing attempts. Default is 5 seconds.
+ #sniffing_delay => 5
+
+ ## Set the address of a forward HTTP proxy.
+ #proxy => $es_proxy
+
+ ##Use this if you must run Elasticsearch behind a proxy that remaps the root path for the Elasticsearch HTTP API lives
+ #path => $es_path
+
+ ######### Elasticsearch request configurations #########
+
+ ## This setting defines the maximum sized bulk request Logstash will make.
+ #flush_size => ?
+
+ ######### Document configurations #########
+
+ index => "logstash-%{+YYYY.MM.dd}"
+ document_type => "logs"
+
+ ## This can be used to associate child documents with a parent using the parent ID.
+ #parent => "abcd'
+ }
+}
+