summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSylvain Desbureaux <sylvain.desbureaux@orange.com>2020-04-22 10:50:26 +0200
committerSylvain Desbureaux <sylvain.desbureaux@orange.com>2020-04-22 10:51:36 +0200
commit8c2a16283583a6e830bdf915bdbfee9f49b8b992 (patch)
treec0f42834ca36ff92a760cea0ea7b498d2b302aff
parentf2556fe72a09071e50ac485dca9137aab9a4c6c5 (diff)
[CLAMP] use common templates for logs
Using the common templates for centralized logging. Issue-ID: OOM-2370 Signed-off-by: Sylvain Desbureaux <sylvain.desbureaux@orange.com> Change-Id: I4824e11bd2be971d15e3a39510f544d2c36b05fd
-rw-r--r--kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml15
-rw-r--r--kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml25
-rw-r--r--kubernetes/clamp/charts/clamp-backend/values.yaml4
-rw-r--r--kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml4
-rw-r--r--kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf6
-rwxr-xr-xkubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh4
-rw-r--r--kubernetes/clamp/resources/config/log/filebeat/filebeat.yml53
-rw-r--r--kubernetes/clamp/templates/configmap.yaml2
-rw-r--r--kubernetes/clamp/templates/deployment.yaml26
-rw-r--r--kubernetes/clamp/values.yaml5
10 files changed, 85 insertions, 59 deletions
diff --git a/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml b/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml
index 0011c6a6d4..f66312c741 100644
--- a/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml
+++ b/kubernetes/clamp/charts/clamp-backend/templates/configmap.yaml
@@ -26,16 +26,5 @@ metadata:
data:
{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }}
spring_application_json: {{ tpl .Values.config.springApplicationJson . | quote }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: {{ include "common.release" . }}-clamp-filebeat-configmap
- namespace: {{ include "common.namespace" . }}
- labels:
- app: {{ include "common.name" . }}
- chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
- release: {{ include "common.release" . }}
- heritage: {{ .Release.Service }}
-data:
-{{ tpl (.Files.Glob "resources/config/log/filebeat/*").AsConfig . | indent 2 }}
+
+{{ include "common.log.configMap" . }}
diff --git a/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml b/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml
index bdae07a261..fa00ffed1a 100644
--- a/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml
+++ b/kubernetes/clamp/charts/clamp-backend/templates/deployment.yaml
@@ -48,17 +48,8 @@ spec:
name: {{ include "common.name" . }}-readiness
containers:
# side car containers
- - name: {{ include "common.name" . }}-filebeat-onap
- image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}"
- imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
- volumeMounts:
- - name: {{ include "common.fullname" . }}-filebeat-conf
- mountPath: /usr/share/filebeat/filebeat.yml
- subPath: filebeat.yml
- - name: {{ include "common.fullname" . }}-data-filebeat
- mountPath: /usr/share/filebeat/data
- - name: {{ include "common.fullname" . }}-logs
- mountPath: /var/log/onap
+ {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.sidecar" . | nindent 8 }}{{ end }}
+ # main container
- name: {{ include "common.name" . }}
image: "{{ include "common.repository" . }}/{{ .Values.image }}"
imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
@@ -81,8 +72,8 @@ spec:
initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
periodSeconds: {{ .Values.readiness.periodSeconds }}
volumeMounts:
- - name: {{ include "common.fullname" . }}-logs
- mountPath: /var/log/onap
+ - name: logs
+ mountPath: {{ .Values.log.path }}
- mountPath: /opt/clamp/sdc-controllers-config.json
name: {{ include "common.fullname" . }}-config
subPath: sdc-controllers-config.json
@@ -109,12 +100,8 @@ spec:
items:
- key: sdc-controllers-config.json
path: sdc-controllers-config.json
- - name: {{ include "common.fullname" . }}-filebeat-conf
- configMap:
- name: {{ include "common.release" . }}-clamp-filebeat-configmap
- - name: {{ include "common.fullname" . }}-data-filebeat
- emptyDir: {}
- - name: {{ include "common.fullname" . }}-logs
+ - name: logs
emptyDir: {}
+ {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.volumes" . | nindent 8 }}{{ end }}
imagePullSecrets:
- name: "{{ include "common.namespace" . }}-docker-registry-key"
diff --git a/kubernetes/clamp/charts/clamp-backend/values.yaml b/kubernetes/clamp/charts/clamp-backend/values.yaml
index 18888547c3..82e4510092 100644
--- a/kubernetes/clamp/charts/clamp-backend/values.yaml
+++ b/kubernetes/clamp/charts/clamp-backend/values.yaml
@@ -33,6 +33,10 @@ pullPolicy: Always
# flag to enable debugging - application support required
debugEnabled: false
+# log configuration
+log:
+ path: /var/log/onap
+
#################################################################
# Application configuration defaults.
#################################################################
diff --git a/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml b/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml
index e4deab0e15..1eb20fce89 100644
--- a/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml
+++ b/kubernetes/clamp/charts/clamp-dash-es/resources/config/elasticsearch.yml
@@ -87,7 +87,7 @@ discovery.zen.minimum_master_nodes: 1
discovery.seed_hosts: []
# # Breaking change in 7.0
# # https://www.elastic.co/guide/en/elasticsearch/reference/7.0/breaking-changes-7.0.html#breaking_70_discovery_changes
-cluster.initial_master_nodes:
+cluster.initial_master_nodes:
- cldash-es-node1
# - docker-test-node-1
# ---------------------------------- Various -----------------------------------
@@ -125,4 +125,4 @@ opendistro_security.check_snapshot_restore_write_privileges: true
opendistro_security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
cluster.routing.allocation.disk.threshold_enabled: false
node.max_local_storage_nodes: 3
-######## End OpenDistro for Elasticsearch Security Demo Configuration ######## \ No newline at end of file
+######## End OpenDistro for Elasticsearch Security Demo Configuration ########
diff --git a/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf
index 4b05910c02..c005fcca3e 100644
--- a/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf
+++ b/kubernetes/clamp/charts/clamp-dash-logstash/resources/config/pipeline.conf
@@ -59,7 +59,7 @@ filter {
if [http_request_failure] or [@metadata][code] != 200 {
mutate {
- add_tag => [ "error" ]
+ add_tag => [ "error" ]
}
}
@@ -195,7 +195,7 @@ filter {
clones => [ "event-cl-aggs" ]
add_tag => [ "event-cl-aggs" ]
}
-
+
if "event-cl-aggs" in [@metadata][request][tags]{
#
# we only need a few fields for aggregations; remove all fields from clone except :
@@ -204,7 +204,7 @@ filter {
prune {
whitelist_names => ["^@.*$","^topic$","^type$","^tags$","^flagFinalFailure$","^flagAbated$","^locationState$","^locationCity$","^vmName$","^vnfName$","^vnfType$","^requestID$","^closedLoopAlarmStart$","^closedLoopControlName$","^closedLoopAlarmEnd$","^target$","^target_type$","^triggerSourceName$","^policyScope$","^policyName$","^policyVersion$"]
}
-
+
}
}
}
diff --git a/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh b/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh
index 224a813db9..2e2ad2e1af 100755
--- a/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh
+++ b/kubernetes/clamp/charts/mariadb/resources/config/mariadb/docker-entrypoint-initdb.d/bootstrap-database.sh
@@ -20,9 +20,9 @@
# limitations under the License.
# ============LICENSE_END============================================
# ===================================================================
-#
+#
###
mysql -uroot -p$MYSQL_ROOT_PASSWORD -f < /docker-entrypoint-initdb.d/bulkload/create-db.sql
-## New model creation
+## New model creation
mysql -uroot -p$MYSQL_ROOT_PASSWORD -f cldsdb4 < /docker-entrypoint-initdb.d/bulkload/create-tables.sql
diff --git a/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml b/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml
new file mode 100644
index 0000000000..dab2e44f5e
--- /dev/null
+++ b/kubernetes/clamp/resources/config/log/filebeat/filebeat.yml
@@ -0,0 +1,53 @@
+# Copyright © 2018 AT&T, Amdocs, Bell Canada Intellectual Property. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+filebeat.prospectors:
+#it is mandatory, in our case it's log
+- input_type: log
+ #This is the canolical path as mentioned in logback.xml, *.* means it will monitor all files in the directory.
+ paths:
+ - /var/log/onap/*/*/*/*.log
+ - /var/log/onap/*/*/*.log
+ - /var/log/onap/*/*.log
+ #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive
+ ignore_older: 48h
+ # Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit
+ clean_inactive: 96h
+
+# Name of the registry file. If a relative path is used, it is considered relative to the
+# data path. Else full qualified file name.
+#filebeat.registry_file: ${path.data}/registry
+
+
+output.logstash:
+ #List of logstash server ip addresses with port number.
+ #But, in our case, this will be the loadbalancer IP address.
+ #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately.
+ hosts: ["{{.Values.config.log.logstashServiceName}}:{{.Values.config.log.logstashPort}}"]
+ #If enable will do load balancing among availabe Logstash, automatically.
+ loadbalance: true
+
+ #The list of root certificates for server verifications.
+ #If certificate_authorities is empty or not set, the trusted
+ #certificate authorities of the host system are used.
+ #ssl.certificate_authorities: $ssl.certificate_authorities
+
+ #The path to the certificate for SSL client authentication. If the certificate is not specified,
+ #client authentication is not available.
+ #ssl.certificate: $ssl.certificate
+
+ #The client certificate key used for client authentication.
+ #ssl.key: $ssl.key
+
+ #The passphrase used to decrypt an encrypted key stored in the configured key file
+ #ssl.key_passphrase: $ssl.key_passphrase
diff --git a/kubernetes/clamp/templates/configmap.yaml b/kubernetes/clamp/templates/configmap.yaml
index 4278a6e6d3..3fce850140 100644
--- a/kubernetes/clamp/templates/configmap.yaml
+++ b/kubernetes/clamp/templates/configmap.yaml
@@ -25,3 +25,5 @@ metadata:
heritage: {{ .Release.Service }}
data:
{{ tpl (.Files.Glob "resources/config/*").AsConfig . | indent 2 }}
+
+{{ include "common.log.configMap" . }}
diff --git a/kubernetes/clamp/templates/deployment.yaml b/kubernetes/clamp/templates/deployment.yaml
index e4ac4723da..e8a7cc25cd 100644
--- a/kubernetes/clamp/templates/deployment.yaml
+++ b/kubernetes/clamp/templates/deployment.yaml
@@ -48,17 +48,8 @@ spec:
name: {{ include "common.name" . }}-readiness
containers:
# side car containers
- - name: {{ include "common.name" . }}-filebeat-onap
- image: "{{ .Values.global.loggingRepository }}/{{ .Values.global.loggingImage }}"
- imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
- volumeMounts:
- - name: {{ include "common.fullname" . }}-filebeat-conf
- mountPath: /usr/share/filebeat/filebeat.yml
- subPath: filebeat.yml
- - name: {{ include "common.fullname" . }}-data-filebeat
- mountPath: /usr/share/filebeat/data
- - name: {{ include "common.fullname" . }}-logs
- mountPath: /var/log/nginx/
+ {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.sidecar" . | nindent 8 }}{{ end }}
+ # main container
- name: {{ include "common.name" . }}
image: "{{ include "common.repository" . }}/{{ .Values.image }}"
imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
@@ -79,8 +70,8 @@ spec:
initialDelaySeconds: {{ .Values.readiness.initialDelaySeconds }}
periodSeconds: {{ .Values.readiness.periodSeconds }}
volumeMounts:
- - name: {{ include "common.fullname" . }}-logs
- mountPath: /var/log/nginx/
+ - name: logs
+ mountPath: {{ .Values.log.path }}
- mountPath: /etc/nginx/conf.d/default.conf
name: {{ include "common.fullname" . }}-config
subPath: default.conf
@@ -101,13 +92,8 @@ spec:
items:
- key: default.conf
path: default.conf
- - name: {{ include "common.fullname" . }}-filebeat-conf
- configMap:
- name: {{ include "common.release" . }}-clamp-filebeat-configmap
- - name: {{ include "common.fullname" . }}-data-filebeat
- emptyDir: {}
- - name: {{ include "common.fullname" . }}-logs
+ - name: logs
emptyDir: {}
+ {{ if .Values.global.centralizedLoggingEnabled }}{{ include "common.log.volumes" . | nindent 8 }}{{ end }}
imagePullSecrets:
- name: "{{ include "common.namespace" . }}-docker-registry-key"
-
diff --git a/kubernetes/clamp/values.yaml b/kubernetes/clamp/values.yaml
index 47eca67f91..0679982740 100644
--- a/kubernetes/clamp/values.yaml
+++ b/kubernetes/clamp/values.yaml
@@ -22,6 +22,7 @@ global: # global defaults
readinessImage: readiness-check:2.0.0
loggingRepository: docker.elastic.co
loggingImage: beats/filebeat:5.5.0
+ centralizedLoggingEnabled: false
subChartsOnly:
enabled: true
@@ -36,6 +37,10 @@ pullPolicy: Always
# flag to enable debugging - application support required
debugEnabled: false
+# log configuration
+log:
+ path: /var/log/nginx/
+
#################################################################
# Application configuration defaults.
#################################################################