Templated logback configuration prototype

Change-Id: I980e50f49643e24a87793ebfe31570cc513ebe45
Signed-off-by: Luke Parker <lparker@amdocs.com>
Issue-ID: LOG-560

6 files changed, 379 insertions, 0 deletions

diff --git a/reference/provider/helm/logback/README.MD b/reference/provider/helm/logback/README.MD
new file mode 100644
index 0000000..4da2d43
--- /dev/null
+++ b/reference/provider/helm/logback/README.MD
@@ -0,0 +1,51 @@
+# Overview
+
+This is an example of logback configuration templated for use with Helm.
+
+It provides:
+
+1. Boilerplate for configuring several appenders:
+    1. Beats - machine-readable, tab-separated, fully escaped file.
+    2. Console - somewhat human-readable (no tab or newline replacement in messages or stacktraces).
+    3. EELF - legacy format, output sorted into several files.
+    4. Syslog - Logstash-compatible network transport.
+2. Values-based activation/deactivation of each appender.
+3. Values-based configuration of root loglevels.
+4. Values-based configuration of individual loggers and their loglevels.
+5. Values-based configuration of queue lengths, rollover, reconfiguration intervals, etc.
+
+# Installation
+
+From ```./charts```, run:
+
+```
+helm install .
+```
+
+# Configuration
+
+Via ```values.yaml```.
+
+* ```componentName``` and ```subcomponentName``` always customized. These determine the output directory for files appenders. (And must therefore be unique per container, and also per deployment).
+* Everything else customized only as necessary.
+
+# Status
+
+Before this can be made available generally available:
+
+1. Discussion:
+    1. Is this the right idea?
+    2. Is this the right way to parameterize logging configuration?
+    3. What can be done better?
+        1. What should be parametrized by isn't?
+        2. What is parameterized needlessly (i.e. providing a degree of configurability we don't want to support)?
+2. Log4j (v1.X and v2.X) equivalents.
+3. Test scenarios in this directory:
+    1. Write some logs within the container.
+    2. Parse the logs to verify that they're as expected.
+    3. Run the same tests for each logging provider.
+
+It doesn't:
+
+1. Provide appender-specific logger-to-level mappings. Each logger gets the same level for each appender.
+2. Define globals, or nececssarily fit into the nascent scheme for OOM helm chart parameterization.
diff --git a/reference/provider/helm/logback/chart/Chart.yaml b/reference/provider/helm/logback/chart/Chart.yaml
new file mode 100644
index 0000000..0795196
--- /dev/null
+++ b/reference/provider/helm/logback/chart/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+name: logdemo
+version: 1.0.0
+description: whatever
diff --git a/reference/provider/helm/logback/chart/resources/logback.xml b/reference/provider/helm/logback/chart/resources/logback.xml
new file mode 100644
index 0000000..eecd13d
--- /dev/null
+++ b/reference/provider/helm/logback/chart/resources/logback.xml
@@ -0,0 +1,282 @@
+<configuration scan='{{.Values.log.scan.enabled}}' scanPeriod='{{.Values.log.scan.seconds}} seconds' debug='{{.Values.log.debug}}'>
+
+    <!-- Component-specific settings. Will ALWAYS change. -->
+
+    <property name="componentName" value='{{default "UNSET_COMPONENT" .Values.log.componentName}}'/>
+    <property name="subcomponentName" value='{{default "UNSET_SUBCOMPONENT" .Values.log.subcomponentName}}'/>
+
+    <!-- Common settings. Will SOMETIMES change. -->
+
+    <property name="serverName" value='{{.Values.log.serverName}}'/>
+    <property name="logDir" value='{{.Values.log.logDir}}'/>
+    <property name="maxFileSize" value='{{.Values.log.maxFileSize}}'/>
+    <property name="maxHistory" value='{{.Values.log.maxHistory}}'/>
+    <property name="totalSizeCap" value='{{.Values.log.totalSizeCap}}'/>
+    <property name="queueSize" value='{{.Values.log.queueSize}}'/>
+
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+
+    <property name="p_tim" value="%d{&quot;yyyy-MM-dd'T'HH:mm:ss.SSSXXX&quot;, UTC}"/>
+    <property name="p_lvl" value="%level"/>
+    <property name="p_log" value="%logger"/>
+    <property name="p_mdc" value="%replace(%replace(%mdc){'\t','\\\\t'}){'\n', '\\\\n'}"/>
+    <property name="p_msg" value="%replace(%replace(%msg){'\t', '\\\\t'}){'\n','\\\\n'}"/>
+    <property name="p_exc" value="%replace(%replace(%rootException){'\t', '\\\\t'}){'\n','\\\\n'}"/>
+    <property name="p_mak" value="%replace(%replace(%marker){'\t', '\\\\t'}){'\n','\\\\n'}"/>
+    <property name="p_thr" value="%thread"/>
+
+    <!-- Appender-specific properties. -->
+
+    <!-- Console (human-readable) logging -->
+
+    <property
+            name="consolePattern"
+            value="%nopexception${p_log}\t${p_tim}\t${p_lvl}\t%message\t${p_mdc}\t%rootException\t${p_mak}\t${p_thr}%n"/>
+
+    <!-- Beats (indexable) output. -->
+
+    <property name="beatsPattern" value="%nopexception${p_tim}\t${p_thr}\t${p_lvl}\t${p_log}\t${p_mdc}\t${p_msg}\t${p_exc}\t${p_mak}\t%n"/>
+    <property name="beatsLogRoot" value="${logDir}/beats"/>
+    <property name="beatsLogDirectory" value="${beatsLogRoot}/${componentName}/${subcomponentName}"/>
+    <property name="beatsLogFilename" value="${serverName}"/>
+
+    <!-- EELF (legacy) output. -->
+
+    <property name="eelfAuditLogName" value="${serverName}-audit"/>
+    <property name="eelfMetricsLogName" value="${serverName}-metrics"/>
+    <property name="eelfDebugLogName" value="${serverName}-debug"/>
+    <property name="eelfErrorLogName" value="${serverName}-error"/>
+    <property name="eelfApplicationLogName" value="${serverName}-application"/>
+    <property
+            name="eelfAuditPattern"
+            value="%X{BeginTimestamp}|%X{EndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread||%X{ServiceName}|%X{PartnerName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{RemoteHost}|%logger|||||||%msg%n"/>
+    <property
+            name="eelfMetricsPattern"
+            value="%X{BeginTimestamp}|%X{EndTimestamp}|%X{RequestId}|%X{ServiceInstanceId}|%thread||%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%X{StatusCode}|%X{ResponseCode}|%X{ResponseDesc}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{Timer}|%X{ServerFQDN}|%X{RemoteHost}|%logger|||%X{TargetVirtualEntity}|||||%msg%n"/>
+    <property
+            name="eelfErrorPattern"
+            value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}|%X{RequestId}|%thread|%X{ServiceName}|%X{PartnerName}|%X{TargetEntity}|%X{TargetServiceName}|%.-5level|%X{ErrorCode}|%X{ErrorDesc}|%msg%n"/>
+    <property
+            name="eelfDebugPattern"
+            value="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX, UTC}|%X{RequestId}|%msg%n"/>
+    <property
+            name="eelfApplicationPattern"
+            value="${eelfErrorPattern}" />
+    <property name="eelfLogRoot" value="${logDir}/eelf"/>
+    <property name="eelfLogDirectory" value="${eelfLogRoot}/${componentName}/${subcomponentName}"/>
+
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+    <!-- /////////////////////////////////////////////////////////////////////////////////////////////////////////////// -->
+
+    <!-- Console output. /////////////////////////////////////////// -->
+
+    <appender class="ch.qos.logback.core.ConsoleAppender" name="Console">
+        <encoder>
+            <pattern>${consolePattern}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="ConsoleAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="Console"/>
+    </appender>
+
+    <!-- Indexable output. /////////////////////////////////////////// -->
+
+    <appender name="Beats" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${beatsLogDirectory}/${beatsLogFilename}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${beatsLogDirectory}/${beatsLogFilename}.%i.log.%d</fileNamePattern>
+            <maxFileSize>${maxFileSize}</maxFileSize>
+            <maxHistory>${maxHistory}</maxHistory>
+            <totalSizeCap>${totalSizeCap}</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>${beatsPattern}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="BeatsAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="Beats"/>
+    </appender>
+
+    <!-- Syslog output. -->
+
+    <appender name="Syslog" class="ch.qos.logback.classic.net.SyslogAppender">
+        <syslogHost>{{default "localhost" .Values.log.syslog.host}}</syslogHost>
+        <port>{{default "514" .Values.log.syslog.port}}</port>
+        <facility>{{default "USER" .Values.log.syslog.facility}}</facility>
+    </appender>
+
+    <appender name="SyslogAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="Syslog"/>
+    </appender>
+
+    <!-- EELF output. -->
+
+    <!-- Audit. -->
+
+    <appender name="EELFAudit" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${eelfLogDirectory}/${eelfAuditLogName}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${eelfLogDirectory}/${eelfAuditLogName}.%i.log.%d</fileNamePattern>
+            <maxFileSize>${maxFileSize}</maxFileSize>
+            <maxHistory>${maxHistory}</maxHistory>
+            <totalSizeCap>${totalSizeCap}</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>${eelfAuditPattern}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="EELFAuditAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="EELFAudit"/>
+    </appender>
+
+    <!-- EELF metrics. -->
+
+    <appender name="EELFMetrics" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${eelfLogDirectory}/${eelfMetricsLogName}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${eelfLogDirectory}/${eelfMetricsLogName}.%i.log.%d</fileNamePattern>
+            <maxFileSize>${maxFileSize}</maxFileSize>
+            <maxHistory>${maxHistory}</maxHistory>
+            <totalSizeCap>${totalSizeCap}</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>${eelfMetricsPattern}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="EELFMetricsAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="EELFMetrics"/>
+    </appender>
+
+    <!-- EELF debug -->
+
+    <appender name="EELFDebug" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${eelfLogDirectory}/${eelfDebugLogName}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${eelfLogDirectory}/${eelfDebugLogName}.%i.log.%d</fileNamePattern>
+            <maxFileSize>${maxFileSize}</maxFileSize>
+            <maxHistory>${maxHistory}</maxHistory>
+            <totalSizeCap>${totalSizeCap}</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>${eelfDebugPattern}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="EELFDebugAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="EELFDebug"/>
+    </appender>
+
+    <!-- EELF error -->
+
+    <appender name="EELFError" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${eelfLogDirectory}/${eelfErrorLogName}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${eelfLogDirectory}/${eelfErrorLogName}.%i.log.%d</fileNamePattern>
+            <maxFileSize>${maxFileSize}</maxFileSize>
+            <maxHistory>${maxHistory}</maxHistory>
+            <totalSizeCap>${totalSizeCap}</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>${eelfErrorPattern}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="EELFErrorAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="EELFError"/>
+    </appender>
+
+    <!-- EELF general -->
+
+    <appender name="EELF" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>${eelfLogDirectory}/${eelfApplicationLogName}.log</file>
+        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+            <fileNamePattern>${eelfLogDirectory}/${eelfApplicationLogName}.%i.log.%d</fileNamePattern>
+            <maxFileSize>${maxFileSize}</maxFileSize>
+            <maxHistory>${maxHistory}</maxHistory>
+            <totalSizeCap>${totalSizeCap}</totalSizeCap>
+        </rollingPolicy>
+        <encoder>
+            <pattern>${eelfApplicationPattern}</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="EELFAsync" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>${queueSize}</queueSize>
+        <appender-ref ref="EELF"/>
+    </appender>
+
+    <!-- ///////////////////////////////////////////////////////////////////// -->
+    <!-- ///////////////////////////////////////////////////////////////////// -->
+    <!-- ///////////////////////////////////////////////////////////////////// -->
+    <!-- ///////////////////////////////////////////////////////////////////// -->
+
+    <!-- EELF loggers. -->
+
+    {{if .Values.log.root.eelf}}
+    <logger name="com.att.eelf.audit" level="info" additivity="false">
+        <appender-ref ref="EELFAuditAsync"/>
+    </logger>
+
+    <logger name="com.att.eelf.metrics" level="info" additivity="false">
+        <appender-ref ref="EELFMetricsAsync"/>
+    </logger>
+
+    <logger name="com.att.eelf.debug" level="debug" additivity="false">
+        <appender-ref ref="EELFDebugAsync"/>
+    </logger>
+
+    <logger name="com.att.eelf.error" level="error" additivity="false">
+        <appender-ref ref="EELFErrorAsync"/>
+    </logger>
+    {{ end }}
+
+    <!-- Root logger. -->
+
+    <root level='{{.Values.log.root.level}}'>
+        {{ if .Values.log.root.beats }}
+        <appender-ref ref="BeatsAsync"/>
+        {{ end }}
+        {{ if .Values.log.root.console }}
+        <appender-ref ref="ConsoleAsync"/>
+        {{ end }}
+        {{ if .Values.log.root.eelf }}
+        <appender-ref ref="EELFAsync"/>
+        {{ end }}
+    </root>
+
+    <!-- Named loggers. -->
+
+    {{- range .Values.log.loggers }}
+    <logger name='{{.name}}' level='{{.level}}' additivity="false">
+        {{ if .beats }}
+        <appender-ref ref="BeatsAsync"/>
+        {{ end }}
+        {{ if .console }}
+        <appender-ref ref="ConsoleAsync"/>
+        {{ end }}
+        {{ if .syslog }}
+        <appender-ref ref="SyslogAsync"/>
+        {{ end }}
+        {{ if .eelf }}
+        <appender-ref ref="EELFAsync"/>
+        {{ end }}
+    </logger>
+    {{- end}}
+
+</configuration>
diff --git a/reference/provider/helm/logback/chart/templates/configmap.yaml b/reference/provider/helm/logback/chart/templates/configmap.yaml
new file mode 100644
index 0000000..0122bd4
--- /dev/null
+++ b/reference/provider/helm/logback/chart/templates/configmap.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: logback-configmap
+  namespace: {{ .Values.nsPrefix }}
+data:
+{{ tpl (.Files.Glob "resources/*").AsConfig . | indent 2 }}
diff --git a/reference/provider/helm/logback/chart/values.yaml b/reference/provider/helm/logback/chart/values.yaml
new file mode 100644
index 0000000..2db0e99
--- /dev/null
+++ b/reference/provider/helm/logback/chart/values.yaml
@@ -0,0 +1,35 @@
+log:
+
+  componentName: c1
+  subcomponentName: s1
+
+  debug: true
+
+  scan:
+    enabled: true
+    seconds: 60
+
+  serverName: ${HOSTNAME}
+  logDir: /var/log/onap
+  maxFileSize: 50MB
+  maxHistory: 50
+  totalSizeCap: 20GB
+  queueSize: 256
+
+  syslog:
+      host: localhost
+      port: 514
+      facility: USER
+
+  root:
+    level: INFO
+    beats: true
+    console: true
+    syslog: false
+    eelf: false
+
+  # loggers:
+  #   - name: org.onap.logging.helm.test
+  #     level: DEBUG
+  #     beats: true
+  #     console: true
diff --git a/reference/provider/helm/logback/test/placeholder.txt b/reference/provider/helm/logback/test/placeholder.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/reference/provider/helm/logback/test/placeholder.txt