Merge "[CLAMP] use common templates for logs"

10 files changed, 85 insertions, 59 deletions

diff --git a/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml b/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml
index 0011c6a6d4..f66312c741 100644
--- a/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml
+++ b/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml
@@ -26,16 +26,5 @@ metadata:
 data:
 {{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }}
   spring_application_json: {{ tpl .Values.config.springApplicationJson . | quote }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ include "common.release" . }}-clamp-filebeat-configmap
-  namespace: {{ include "common.namespace" . }}
-  labels:
-    app: {{ include "common.name" . }}
-    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
-    release: {{ include "common.release" . }}
-    heritage: {{ .Release.Service }}
-data:
-{{ tpl (.Files.Glob "resources/config/log/filebeat/*").AsConfig . | indent 2 }}
+
+{{ include "common.log.configMap" . }}
diff --git a/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml b/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml
index bdae07a261..fa00ffed1a 100644
--- a/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml
+++ b/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml
@@ -48,17 +48,8 @@ spec:
         name: {{ include "common.name" . }}-readiness
       containers:
         # side car containers
-        - name: {{ include "common.name" . }}-filebeat-onap
-          image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}"
-          imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
-          volumeMounts:
-          - name: {{ include "common.fullname" . }}-filebeat-conf
-            mountPath: /usr/share/filebeat/filebeat.yml
-            subPath: filebeat.yml
-          - name: {{ include "common.fullname" . }}-data-filebeat
-            mountPath: /usr/share/filebeat/data
-          - name: {{ include "common.fullname" . }}-logs
-            mountPath: /var/log/onap
+        {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.sidecar" . | nindent 8 }}{{ end }}
+        # main container
         - name: {{ include "common.name" . }}
           image: "{{ include "common.repository" . }}/{{ .Values.image }}"
           imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
@@ -81,8 +72,8 @@ spec:
             initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
             periodSeconds: {{ .Values.readiness.periodSeconds }}
           volumeMounts:
-          - name: {{ include "common.fullname" . }}-logs
-            mountPath: /var/log/onap
+          - name: logs
+            mountPath: {{ .Values.log.path }}
           - mountPath: /opt/clamp/sdc-controllers-config.json
             name: {{ include "common.fullname" . }}-config
             subPath: sdc-controllers-config.json
@@ -109,12 +100,8 @@ spec:
             items:
             - key: sdc-controllers-config.json
               path: sdc-controllers-config.json
-        - name: {{ include "common.fullname" . }}-filebeat-conf
-          configMap:
-            name: {{ include "common.release" . }}-clamp-filebeat-configmap
-        - name: {{ include "common.fullname" . }}-data-filebeat
-          emptyDir: {}
-        - name:  {{ include "common.fullname" . }}-logs
+        - name:  logs
           emptyDir: {}
+        {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.volumes" . | nindent 8 }}{{ end }}
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
diff --git a/kubernetes/clamp/charts/clamp-backend/values.yaml b/kubernetes/clamp/charts/clamp-backend/values.yaml
index 48252176f9..ca444aa175 100644
--- a/kubernetes/clamp/charts/clamp-backend/values.yaml
+++ b/kubernetes/clamp/charts/clamp-backend/values.yaml
@@ -33,6 +33,10 @@ pullPolicy: Always
 # flag to enable debugging - application support required
 debugEnabled: false
 
+# log configuration
+log:
+  path: /var/log/onap
+
 #################################################################
 # Application configuration defaults.
 #################################################################
diff --git a/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml b/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml
index e4deab0e15..1eb20fce89 100644
--- a/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml
+++ b/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml
@@ -87,7 +87,7 @@ discovery.zen.minimum_master_nodes: 1
 discovery.seed_hosts: []
 # # Breaking change in 7.0
 # # https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html#breaking_70_discovery_changes
-cluster.initial_master_nodes: 
+cluster.initial_master_nodes:
     - cldash-es-node1
 #    - docker-test-node-1
 # ---------------------------------- Various -----------------------------------
@@ -125,4 +125,4 @@ opendistro_security.check_snapshot_restore_write_privileges: true
 opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
 cluster.routing.allocation.disk.threshold_enabled: false
 node.max_local_storage_nodes: 3
-######## End OpenDistro for Elasticsearch Security Demo Configuration ########
\ No newline at end of file
+######## End OpenDistro for Elasticsearch Security Demo Configuration ########
diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf
index 4b05910c02..c005fcca3e 100644
--- a/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf
+++ b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf
@@ -59,7 +59,7 @@ filter {
 
     if [http_request_failure] or [@metadata][code] != 200 {
        mutate {
-       	   add_tag => [ "error" ]
+              add_tag => [ "error" ]
        }
     }
 
@@ -195,7 +195,7 @@ filter {
             clones => [ "event-cl-aggs" ]
             add_tag => [ "event-cl-aggs" ]
         }
-        
+
         if  "event-cl-aggs" in [@metadata][request][tags]{
             #
             # we only need a few fields for aggregations; remove all fields from clone except :
@@ -204,7 +204,7 @@ filter {
             prune {
                 whitelist_names => ["^@.*$","^topic$","^type$","^tags$","^flagFinalFailure$","^flagAbated$","^locationState$","^locationCity$","^vmName$","^vnfName$","^vnfType$","^requestID$","^closedLoopAlarmStart$","^closedLoopControlName$","^closedLoopAlarmEnd$","^target$","^target_type$","^triggerSourceName$","^policyScope$","^policyName$","^policyVersion$"]
             }
-        
+
         }
     }
 }
diff --git a/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh b/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh
index 224a813db9..2e2ad2e1af 100755
--- a/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh
+++ b/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh
@@ -20,9 +20,9 @@
 # limitations under the License.
 # ============LICENSE_END============================================
 # ===================================================================
-# 
+#
 ###
 
 mysql -uroot -p$MYSQL_ROOT_PASSWORD -f < /docker-entrypoint-initdb.d/bulkload/create-db.sql
-## New model creation 
+## New model creation
 mysql -uroot -p$MYSQL_ROOT_PASSWORD -f cldsdb4 < /docker-entrypoint-initdb.d/bulkload/create-tables.sql
diff --git a/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml b/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml
new file mode 100644
index 0000000000..dab2e44f5e
--- /dev/null
+++ b/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml
@@ -0,0 +1,53 @@
+# Copyright © 2018  AT&T, Amdocs, Bell Canada Intellectual Property.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+filebeat.prospectors:
+#it is mandatory, in our case it's log
+- input_type: log
+  #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory.
+  paths:
+    - /var/log/onap/*/*/*/*.log
+    - /var/log/onap/*/*/*.log
+    - /var/log/onap/*/*.log
+  #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive
+  ignore_older: 48h
+  # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit
+  clean_inactive: 96h
+
+# Name of the registry file. If a relative path is used, it is considered relative to the
+# data path. Else full qualified file name.
+#filebeat.registry_file: ${path.data}/registry
+
+
+output.logstash:
+  #List of logstash server ip addresses with port number.
+  #But, in our case, this will be the loadbalancer IP address.
+  #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately.
+  hosts: ["{{.Values.config.log.logstashServiceName}}:{{.Values.config.log.logstashPort}}"]
+  #If enable will do load balancing among availabe Logstash, automatically.
+  loadbalance: true
+
+  #The list of root certificates for server verifications.
+  #If certificate_authorities is empty or not set, the trusted
+  #certificate authorities of the host system are used.
+  #ssl.certificate_authorities: $ssl.certificate_authorities
+
+  #The path to the certificate for SSL client authentication. If the certificate is not specified,
+  #client authentication is not available.
+  #ssl.certificate: $ssl.certificate
+
+  #The client certificate key used for client authentication.
+  #ssl.key: $ssl.key
+
+  #The passphrase used to decrypt an encrypted key stored in the configured key file
+  #ssl.key_passphrase: $ssl.key_passphrase
diff --git a/kubernetes/clamp/templates/configmap.yaml b/kubernetes/clamp/templates/configmap.yaml
index 4278a6e6d3..3fce850140 100644
--- a/kubernetes/clamp/templates/configmap.yaml
+++ b/kubernetes/clamp/templates/configmap.yaml
@@ -25,3 +25,5 @@ metadata:
     heritage: {{ .Release.Service }}
 data:
 {{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }}
+
+{{ include "common.log.configMap" . }}
diff --git a/kubernetes/clamp/templates/deployment.yaml b/kubernetes/clamp/templates/deployment.yaml
index e4ac4723da..e8a7cc25cd 100644
--- a/kubernetes/clamp/templates/deployment.yaml
+++ b/kubernetes/clamp/templates/deployment.yaml
@@ -48,17 +48,8 @@ spec:
         name: {{ include "common.name" . }}-readiness
       containers:
         # side car containers
-        - name: {{ include "common.name" . }}-filebeat-onap
-          image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}"
-          imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
-          volumeMounts:
-          - name: {{ include "common.fullname" . }}-filebeat-conf
-            mountPath: /usr/share/filebeat/filebeat.yml
-            subPath: filebeat.yml
-          - name: {{ include "common.fullname" . }}-data-filebeat
-            mountPath: /usr/share/filebeat/data
-          - name: {{ include "common.fullname" . }}-logs
-            mountPath: /var/log/nginx/
+        {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.sidecar" . | nindent 8 }}{{ end }}
+        # main container
         - name: {{ include "common.name" . }}
           image: "{{ include "common.repository" . }}/{{ .Values.image }}"
           imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
@@ -79,8 +70,8 @@ spec:
             initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
             periodSeconds: {{ .Values.readiness.periodSeconds }}
           volumeMounts:
-          - name: {{ include "common.fullname" . }}-logs
-            mountPath: /var/log/nginx/
+          - name: logs
+            mountPath: {{ .Values.log.path }}
           - mountPath: /etc/nginx/conf.d/default.conf
             name: {{ include "common.fullname" . }}-config
             subPath: default.conf
@@ -101,13 +92,8 @@ spec:
             items:
             - key: default.conf
               path: default.conf
-        - name: {{ include "common.fullname" . }}-filebeat-conf
-          configMap:
-            name: {{ include "common.release" . }}-clamp-filebeat-configmap
-        - name: {{ include "common.fullname" . }}-data-filebeat
-          emptyDir: {}
-        - name:  {{ include "common.fullname" . }}-logs
+        - name:  logs
           emptyDir: {}
+        {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.volumes" . | nindent 8 }}{{ end }}
       imagePullSecrets:
       - name: "{{ include "common.namespace" . }}-docker-registry-key"
-
diff --git a/kubernetes/clamp/values.yaml b/kubernetes/clamp/values.yaml
index 47eca67f91..0679982740 100644
--- a/kubernetes/clamp/values.yaml
+++ b/kubernetes/clamp/values.yaml
@@ -22,6 +22,7 @@ global: # global defaults
   readinessImage: readiness-check:2.0.0
   loggingRepository: docker.elastic.co
   loggingImage: beats/filebeat:5.5.0
+  centralizedLoggingEnabled: false
 
 subChartsOnly:
   enabled: true
@@ -36,6 +37,10 @@ pullPolicy: Always
 # flag to enable debugging - application support required
 debugEnabled: false
 
+# log configuration
+log:
+  path: /var/log/nginx/
+
 #################################################################
 # Application configuration defaults.
 #################################################################