[LOG-31]Config for FileBeat Shipping of ONAP Logs

Change-Id: I176067a1ad84aba0fa99a609e695a77abb118cc8
Signed-off-by: alkac <alkac@amdocs.com>

7 files changed, 202 insertions, 0 deletions

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..4574472
--- /dev/null
+++ b/README.md
@@ -0,0 +1,2 @@
+This folder contains reference configurations for ONAP logging providers and an Elastic Stack pipeline consisting of Filebeat, Logstash, Elastic Search and Kibana Discover. 
+See the OOM project for automated deployment of the Elastic Stack pipeline and application of provider configuration.
diff --git a/elasticstack/filebeat/log4j/conf/README.md b/elasticstack/filebeat/log4j/conf/README.md
new file mode 100644
index 0000000..e9fd228
--- /dev/null
+++ b/elasticstack/filebeat/log4j/conf/README.md
@@ -0,0 +1,35 @@
+#Filebeat canonical configuration
+----------------------------------
+This filebeat configuration is for ONAP components which uses log4j 1.2.X. The components that uses log4j are -
+- APPC
+- SDNC
+- DCAE
+
+#File(s) provided for filebeat deployment
+-----------------------------------------
+1. filebeat.yml
+
+#Instructions for Deployment
+----------------------------
+1. Create path /etc/onap/filebeat/conf.d/log4j on the host on which the filebeat has to be installed.
+2. The provided filebeat.yml is parameterized and has tokens instead of values for certain deployment specific parameters (like host). These tokens have a syntax as '$[a-zA-Z_]+'. 
+3. The tokens are listed in another file called filebeat.properties. These properties are also provided with commented description about them in the file itself. These tokens have to be replaced with the appropriate values as per the deployment environment setup before deployment. 
+4. Deployment script should replace the tokens in the filebeat.yml with the values specified in filebeat.properties file.
+5. Now, save the final filebeat.yml at location created in step 1.
+6. Create path /var/log/onap on the host.
+7. Create path /usr/share/onap/filebeat/data on the host. Provide it with write permissions for "other" users.
+8. Following is the list of specifications for filebeat container creation -
+
+   Image        - 'docker.elastic.co/beats/filebeat:5.5.0' available in the Elastic Docker Registry.
+
+   Volume mount - The filebeat container must have two host paths mapped as volume in the container
+                  1. Host path -      /etc/onap/filebeat/conf.d/log4j/filebeat.yml   mapped to
+                     Container path - /usr/share/filebeat/filebeat.yml
+
+                  2. Host path -      /var/log/onap      mapped to
+                     Container path - /var/log/onap
+
+                  3. Host path -      /usr/share/onap/filebeat/data      mapped to
+                     Container path - /usr/share/filebeat/data
+
+9. The third volume mapping is specific to the particular filebeat on that host. This path i.e. (/usr/share/onap/filebeat/data/) should not be shared and its contents should be unique to that host.
\ No newline at end of file
diff --git a/elasticstack/filebeat/log4j/conf/filebeat.properties b/elasticstack/filebeat/log4j/conf/filebeat.properties
new file mode 100644
index 0000000..db5c70a
--- /dev/null
+++ b/elasticstack/filebeat/log4j/conf/filebeat.properties
@@ -0,0 +1,20 @@
+
+#List of logstash server ip addresses with port number or a loadbalancer IP address. 
+#In our case, this will be the loadbalancer IP address.
+hosts = ["loadbalancer.onap:5044"]
+#hosts = ["10.247.86.59:5044"]
+
+#The list of root certificates for server verifications. 
+#If certificate_authorities is empty or not set, the trusted 
+#certificate authorities of the host system are used.
+#ssl.certificate_authorities = ["/etc/pki/root/ca.pem"]
+  
+#The path to the certificate for SSL client authentication. If the certificate is not specified, 
+#client authentication is not available.
+#ssl.certificate = "/etc/pki/client/cert.pem"
+  
+#The client certificate key used for client authentication. 
+#ssl.key = "/etc/pki/client/cert.key"
+  
+#The passphrase used to decrypt an encrypted key stored in the configured key file
+#ssl.key_passphrase = ''
\ No newline at end of file
diff --git a/elasticstack/filebeat/log4j/conf/filebeat.yml b/elasticstack/filebeat/log4j/conf/filebeat.yml
new file mode 100644
index 0000000..1fa01e5
--- /dev/null
+++ b/elasticstack/filebeat/log4j/conf/filebeat.yml
@@ -0,0 +1,47 @@
+filebeat.prospectors:
+#it is mandatory, in our case it's log
+- input_type: log
+  #This is the canonical path as mentioned in logback.xml, *.* means it will monitor all files in the directory.
+  paths:
+    - /var/log/onap/*/*/*.log
+    - /var/log/onap/*/*.log
+    
+  #The below commented properties are for time-based rolling policy. But as the log4j 1.2x does not support time-based rolling these properties are not set
+  #Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive
+  #ignore_older: 48h
+  #Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records within limit
+  #clean_inactive: 96h
+  
+  #Multiline properties for log4j xml log events
+  multiline.pattern: '</log4j:event>'
+  multiline.negate: true
+  multiline.match: before
+  #multiline.max_lines: 500
+  #multiline.timeout: 5s
+   
+#Name of the registry file. If a relative path is used, it is considered relative to the data path. Else full qualified file name.
+#filebeat.registry_file: ${path.data}/registry
+
+
+output.logstash:
+  #List of logstash server ip addresses with port number. 
+  #But, in our case, this will be the loadbalancer IP address.
+  #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately.
+  hosts: ["loadbalancer.onap:5044"]
+  #If enable will do load balancing among availabe Logstash, automatically.
+  loadbalance: true
+  
+  #The list of root certificates for server verifications. 
+  #If certificate_authorities is empty or not set, the trusted 
+  #certificate authorities of the host system are used.
+  #ssl.certificate_authorities: $ssl.certificate_authorities
+  
+  #The path to the certificate for SSL client authentication. If the certificate is not specified, 
+  #client authentication is not available.
+  #ssl.certificate: $ssl.certificate
+  
+  #The client certificate key used for client authentication. 
+  #ssl.key: $ssl.key
+  
+  #The passphrase used to decrypt an encrypted key stored in the configured key file
+  #ssl.key_passphrase: $ssl.key_passphrase
diff --git a/elasticstack/filebeat/logback/conf/README.md b/elasticstack/filebeat/logback/conf/README.md
new file mode 100644
index 0000000..37182e5
--- /dev/null
+++ b/elasticstack/filebeat/logback/conf/README.md
@@ -0,0 +1,38 @@
+#Filebeat canonical configuration
+----------------------------------
+This filebeat configuration is for ONAP components which uses logback. The components that uses logback are -
+- MSO
+- SDC
+- AAI
+- Policy
+- Portal
+- VID
+
+#File(s) provided for filebeat deployment
+-----------------------------------------
+1. filebeat.yml
+
+#Instructions for Deployment
+----------------------------
+1. Create path /etc/onap/filebeat/conf.d/logback on the host on which the filebeat has to be installed.
+2. The provided filebeat.yml is parameterized and has tokens instead of values for certain deployment specific parameters (like host). These tokens have a syntax as '$[a-zA-Z_]+'. 
+3. The tokens are listed in another file called filebeat.properties. These properties are also provided with commented description about them in the file itself. These tokens have to be replaced with the appropriate values as per the deployment environment setup before deployment. 
+4. Deployment script should replace the tokens in the filebeat.yml with the values specified in filebeat.properties file.
+5. Now, save the final filebeat.yml at location created in step 1.
+6. Create path /var/log/onap on the host.
+7. Create path /usr/share/onap/filebeat/data on the host. Provide it with write permissions for "other" users.
+8. Following is the list of specifications for filebeat container creation-
+
+   Image        - 'docker.elastic.co/beats/filebeat:5.5.0' available in the Elastic Docker Registry.
+
+   Volume mount - The filebeat container must have two host paths mapped as volume in the container
+                  1. Host path -      /etc/onap/filebeat/conf.d/logback/filebeat.yml   mapped to
+                     Container path - /usr/share/filebeat/filebeat.yml
+
+                  2. Host path -      /var/log/onap      mapped to
+                     Container path - /var/log/onap
+
+                  3. Host path -      /usr/share/onap/filebeat/data      mapped to
+                     Container path - /usr/share/filebeat/data
+
+9. The third volume mapping is specific to the particular filebeat on that host. This path i.e.(/usr/share/onap/filebeat/data/) should not be shared and its contents should be unique to that host.
\ No newline at end of file
diff --git a/elasticstack/filebeat/logback/conf/filebeat.properties b/elasticstack/filebeat/logback/conf/filebeat.properties
new file mode 100644
index 0000000..a150f4e
--- /dev/null
+++ b/elasticstack/filebeat/logback/conf/filebeat.properties
@@ -0,0 +1,21 @@
+
+#List of logstash server ip addresses with port number or a loadbalancer IP address. 
+#In our case, this will be the loadbalancer IP address.
+hosts = ["loadbalancer.onap:5044"]
+#hosts = ["10.247.86.59:5044"]
+
+
+#The list of root certificates for server verifications. 
+#If certificate_authorities is empty or not set, the trusted 
+#certificate authorities of the host system are used.
+#ssl.certificate_authorities = ["/etc/pki/root/ca.pem"]
+  
+#The path to the certificate for SSL client authentication. If the certificate is not specified, 
+#client authentication is not available.
+#ssl.certificate = "/etc/pki/client/cert.pem"
+  
+#The client certificate key used for client authentication. 
+#ssl.key = "/etc/pki/client/cert.key"
+  
+#The passphrase used to decrypt an encrypted key stored in the configured key file
+#ssl.key_passphrase = ''
\ No newline at end of file
diff --git a/elasticstack/filebeat/logback/conf/filebeat.yml b/elasticstack/filebeat/logback/conf/filebeat.yml
new file mode 100644
index 0000000..cfc7859
--- /dev/null
+++ b/elasticstack/filebeat/logback/conf/filebeat.yml
@@ -0,0 +1,39 @@
+filebeat.prospectors:
+#it is mandatory, in our case it's log
+- input_type: log
+  #This is the canonical path as mentioned in logback.xml, *.* means it will monitor all files in the directory.
+  paths:
+    - /var/log/onap/*/*/*.log
+    - /var/log/onap/*/*.log
+  #Files older than this should be ignored. In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive
+  ignore_older: 48h
+  #Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records within limit
+  clean_inactive: 96h
+   
+   
+#Name of the registry file. If a relative path is used, it is considered relative to the data path. Else full qualified file name.
+#filebeat.registry_file: ${path.data}/registry
+
+
+output.logstash:
+  #List of logstash server ip addresses with port number. 
+  #But, in our case, this will be the loadbalancer IP address.
+  #For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately.
+  hosts: ["loadbalancer.onap:5044"]
+  #If enable will do load balancing among availabe Logstash, automatically.
+  loadbalance: true
+  
+  #The list of root certificates for server verifications. 
+  #If certificate_authorities is empty or not set, the trusted 
+  #certificate authorities of the host system are used.
+  #ssl.certificate_authorities: $ssl.certificate_authorities
+  
+  #The path to the certificate for SSL client authentication. If the certificate is not specified, 
+  #client authentication is not available.
+  #ssl.certificate: $ssl.certificate
+  
+  #The client certificate key used for client authentication. 
+  #ssl.key: $ssl.key
+  
+  #The passphrase used to decrypt an encrypted key stored in the configured key file
+  #ssl.key_passphrase: $ssl.key_passphrase
\ No newline at end of file