aboutsummaryrefslogtreecommitdiffstats
path: root/kubernetes/dmaap/components/dmaap-dr-node/resources/config
diff options
context:
space:
mode:
authorefiacor <fiachra.corcoran@est.tech>2019-08-28 21:20:22 +0000
committerefiacor <fiachra.corcoran@est.tech>2019-08-28 21:20:22 +0000
commit7b9a087af97652471e6239916442c9953c5deecf (patch)
treefd407e07bfce19a4e943398215e0f5d5e1c66795 /kubernetes/dmaap/components/dmaap-dr-node/resources/config
parent424278d0f109f5515393d83ea03162b073773e55 (diff)
DMaaP DR refactoring
# Increase docker images # Refactor global vars # Fix logstash paths etc # Remove default feed script Change-Id: Ib15fecc57d6f7f2d4049eff7903c08bc3adb78da Issue-ID: DMAAP-1283 Signed-off-by: efiacor <fiachra.corcoran@est.tech>
Diffstat (limited to 'kubernetes/dmaap/components/dmaap-dr-node/resources/config')
-rw-r--r--kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/createFeed.sh14
-rw-r--r--kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/dedicatedFeed.json18
-rw-r--r--kubernetes/dmaap/components/dmaap-dr-node/resources/config/log/filebeat/filebeat.yml4
-rw-r--r--kubernetes/dmaap/components/dmaap-dr-node/resources/config/logback.xml170
-rw-r--r--kubernetes/dmaap/components/dmaap-dr-node/resources/config/node.properties4
5 files changed, 126 insertions, 84 deletions
diff --git a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/createFeed.sh b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/createFeed.sh
deleted file mode 100644
index 1289ecae86..0000000000
--- a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/createFeed.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-dr_prov_url="{{.Values.config.dmaapDrProv.name}}:{{.Values.config.dmaapDrProv.internalPort2}}"
-ct_header="Content-Type:application/vnd.dmaap-dr.feed"
-obo_header="X-DMAAP-DR-ON-BEHALF-OF:dradmin"
-feed_payload=/opt/app/datartr/etc/dedicatedFeed.json
-
-sleep 20
-
-if curl -k https://${dr_prov_url}/internal/prov | awk 'BEGIN{ORS=""} {print}' | egrep "\"feeds\":\s+\[\]"; then
- curl -X POST -H ${ct_header} -H ${obo_header} --data-ascii @${feed_payload} --post301 --location-trusted -k https://${dr_prov_url};
-else
- echo "NO feed creation required";
-fi
diff --git a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/dedicatedFeed.json b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/dedicatedFeed.json
deleted file mode 100644
index a96f46d528..0000000000
--- a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/feeds/dedicatedFeed.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "name": "Default PM Feed",
- "version": "m1.0",
- "description": "Default feed provisioned for PM File collector",
- "business_description": "Default Feed",
- "suspend": false,
- "deleted": false,
- "changeowner": true,
- "authorization": {
- "classification": "unclassified",
- "endpoint_addrs": [],
- "endpoint_ids": [
- {
- "password": "dradmin",
- "id": "dradmin"
- }]
- }
-} \ No newline at end of file
diff --git a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/log/filebeat/filebeat.yml b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/log/filebeat/filebeat.yml
index f1cf7394b6..e0cb1dd21b 100644
--- a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/log/filebeat/filebeat.yml
+++ b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/log/filebeat/filebeat.yml
@@ -16,6 +16,7 @@
# SPDX-License-Identifier: Apache-2.0
# ============LICENSE_END=========================================================
+# dmaap-dr-node filebeat.yml
filebeat.prospectors:
#it is mandatory, in our case it's log
- input_type: log
@@ -24,7 +25,6 @@ filebeat.prospectors:
- /var/log/onap/*/*/*/*.log
- /var/log/onap/*/*/*.log
- /var/log/onap/*/*.log
- - /opt/app/datartr/logs/*.log
#Files older than this should be ignored.In our case it will be 48 hours i.e. 2 days. It is a helping flag for clean_inactive
ignore_older: 48h
# Remove the registry entry for a file that is more than the specified time. In our case it will be 96 hours, i.e. 4 days. It will help to keep registry records with in limit
@@ -40,7 +40,7 @@ output.logstash:
#List of logstash server ip addresses with port number.
#But, in our case, this will be the loadbalancer IP address.
#For the below property to work the loadbalancer or logstash should expose 5044 port to listen the filebeat events or port in the property should be changed appropriately.
- hosts: ["{{.Values.config.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.config.logstashPort}}"]
+ hosts: ["{{.Values.global.logstashServiceName}}.{{.Release.Namespace}}:{{.Values.global.logstashPort}}"]
#If enable will do load balancing among available Logstash, automatically.
loadbalance: true
diff --git a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/logback.xml b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/logback.xml
index 25a2c6e093..ba247e151c 100644
--- a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/logback.xml
+++ b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/logback.xml
@@ -17,18 +17,21 @@
SPDX-License-Identifier: Apache-2.0
============LICENSE_END=========================================================
-->
-<configuration scan="true" scanPeriod="3 seconds" debug="true">
+<configuration scan="true" scanPeriod="3 seconds" debug="false">
- <property name="generalLogName" value="apicalls" />
- <property name="errorLogName" value="errors" />
- <property name="jettyAndNodeLogName" value="node"/>
+ <property name="logDir" value="/var/log/onap/datarouter" />
+ <!-- log file names -->
+ <property name="auditLog" value="audit" />
+ <property name="errorLog" value="error" />
+ <property name="debugLog" value="debug" />
+ <property name="metricsLog" value="metrics" />
+ <property name="jettyLog" value="jetty" />
- <property name="defaultPattern" value="%d{MM/dd-HH:mm:ss.SSS}|%logger|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{ServiceName}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ServerFQDN}|%X{RemoteHost}|%X{Timer}|%msg%n" />
- <property name="jettyAndNodeLoggerPattern" value="%d{MM/dd-HH:mm:ss.SSS}|%logger|%thread|%.-5level|%msg%n" />
- <property name="debugLoggerPattern" value="%d{MM/dd-HH:mm:ss.SSS}|%X{RequestId}|%X{ServiceInstanceId}|%thread|%X{ServiceName}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ServerFQDN}|%X{RemoteHost}|%X{Timer}|[%caller{3}]|%msg%n" />
-
- <property name="logDirectory" value="/opt/app/datartr/logs" />
+ <!-- log file names -->
+ <property name="defaultPattern" value="%d{MM/dd-HH:mm:ss.SSS}|%logger|%X{RequestId}|%X{InvocationId}|%X{ServiceInstanceId}|%thread|%X{ServiceName}|%X{InstanceUUID}|%.-5level|%X{AlertSeverity}|%X{ServerIPAddress}|%X{ServerFQDN}|%X{RemoteHost}|%X{Timer}%n|%msg%n" />
+ <property name="logDirectory" value="${logDir}" />
+ <!-- Example evaluator filter applied against console appender -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>${defaultPattern}</pattern>
@@ -39,110 +42,181 @@
<!-- EELF Appenders -->
<!-- ============================================================================ -->
- <appender name="EELF"
+ <!-- The EELFAppender is used to record events to the general application
+ log -->
+
+
+ <appender name="Audit"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${logDirectory}/${generalLogName}.log</file>
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>INFO</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
+ <file>${logDirectory}/${auditLog}.log</file>
+ <filter class="org.onap.dmaap.datarouter.node.eelf.AuditFilter">
</filter>
<rollingPolicy
class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${logDirectory}/${generalLogName}.%i.log.zip
+ <fileNamePattern>${logDirectory}/${auditLog}.%i.log.zip
</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>9</maxIndex>
</rollingPolicy>
<triggeringPolicy
class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>5MB</maxFileSize>
+ <maxFileSize>50MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>${defaultPattern}</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="asyncAudit" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="Audit" />
+ </appender>
+
+ <!-- ============================================================================ -->
+
+ <appender name="Metrics"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${metricsLog}.log</file>
+ <filter class="org.onap.dmaap.datarouter.node.eelf.MetricsFilter">
+ </filter>
+ <rollingPolicy
+ class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${logDirectory}/${metricsLog}.%i.log.zip
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
+ class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>50MB</maxFileSize>
</triggeringPolicy>
<encoder>
<pattern>${defaultPattern}</pattern>
</encoder>
</appender>
- <appender name="asyncEELF" class="ch.qos.logback.classic.AsyncAppender">
+ <appender name="asyncMetrics" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>256</queueSize>
- <appender-ref ref="EELF" />
+ <appender-ref ref="Metrics" />
</appender>
- <appender name="EELFError"
+ <!-- ============================================================================ -->
+
+
+ <appender name="Debug"
+ class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>${logDirectory}/${debugLog}.log</file>
+ <filter class="org.onap.dmaap.datarouter.node.eelf.DebugFilter">
+ </filter>
+ <rollingPolicy
+ class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+ <fileNamePattern>${logDirectory}/${debugLog}.%i.log.zip
+ </fileNamePattern>
+ <minIndex>1</minIndex>
+ <maxIndex>9</maxIndex>
+ </rollingPolicy>
+ <triggeringPolicy
+ class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+ <maxFileSize>50MB</maxFileSize>
+ </triggeringPolicy>
+ <encoder>
+ <pattern>${defaultPattern}</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="asyncDebug" class="ch.qos.logback.classic.AsyncAppender">
+ <queueSize>256</queueSize>
+ <appender-ref ref="Debug" />
+ </appender>
+
+ <!-- ============================================================================ -->
+
+ <appender name="Error"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${logDirectory}/${errorLogName}.log</file>
- <filter class="ch.qos.logback.classic.filter.LevelFilter">
- <level>ERROR</level>
- <onMatch>ACCEPT</onMatch>
- <onMismatch>DENY</onMismatch>
+ <file>${logDirectory}/${errorLog}.log</file>
+ <filter class="org.onap.dmaap.datarouter.node.eelf.ErrorFilter">
</filter>
<rollingPolicy
class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${logDirectory}/${errorLogName}.%i.log.zip
+ <fileNamePattern>${logDirectory}/${errorLog}.%i.log.zip
</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>9</maxIndex>
</rollingPolicy>
<triggeringPolicy
class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>5MB</maxFileSize>
+ <maxFileSize>50MB</maxFileSize>
</triggeringPolicy>
<encoder>
<pattern>${defaultPattern}</pattern>
</encoder>
</appender>
- <appender name="asyncEELFError" class="ch.qos.logback.classic.AsyncAppender">
+ <appender name="asyncError" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>256</queueSize>
- <appender-ref ref="EELFError"/>
+ <appender-ref ref="Error"/>
</appender>
<!-- ============================================================================ -->
- <appender name="jettyAndNodelog"
+ <appender name="Jetty"
class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>${logDirectory}/${jettyAndNodeLogName}.log</file>
- <filter class="org.onap.dmaap.datarouter.node.eelf.EELFFilter" />
+ <file>${logDirectory}/${jettyLog}.log</file>
+ <filter class="org.onap.dmaap.datarouter.node.eelf.JettyFilter" />
<rollingPolicy
class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>${logDirectory}/${jettyAndNodeLogName}.%i.log.zip
+ <fileNamePattern>${logDirectory}/${jettyLog}.%i.log.zip
</fileNamePattern>
<minIndex>1</minIndex>
<maxIndex>9</maxIndex>
</rollingPolicy>
<triggeringPolicy
class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>5MB</maxFileSize>
+ <maxFileSize>50MB</maxFileSize>
</triggeringPolicy>
<encoder>
- <pattern>${jettyAndNodeLoggerPattern}</pattern>
+ <pattern>${defaultPattern}</pattern>
</encoder>
</appender>
- <appender name="asyncEELFjettyAndNodelog" class="ch.qos.logback.classic.AsyncAppender">
+ <appender name="asyncJettyLog" class="ch.qos.logback.classic.AsyncAppender">
<queueSize>256</queueSize>
- <appender-ref ref="jettyAndNodelog" />
+ <appender-ref ref="Jetty" />
<includeCallerData>true</includeCallerData>
</appender>
+ <!-- ============================================================================ -->
+
+
<!-- ============================================================================ -->
<!-- EELF loggers -->
<!-- ============================================================================ -->
<logger name="com.att.eelf" level="info" additivity="false">
- <appender-ref ref="asyncEELF" />
+ <appender-ref ref="asyncAudit" />
+ </logger>
+
+ <logger name="com.att.eelf" additivity="false">
+ <appender-ref ref="asyncMetrics" />
+ </logger>
+
+ <logger name="com.att.eelf" additivity="false">
+ <appender-ref ref="asyncDebug" />
</logger>
- <logger name="com.att.eelf.error" level="error" additivity="false">
- <appender-ref ref="asyncEELFError" />
- </logger>
+ <logger name="com.att.eelf.error" additivity="false">
+ <appender-ref ref="asyncError" />
+ </logger>
+
+ <logger name="log4j.logger.org.eclipse.jetty" additivity="false">
+ <appender-ref ref="asyncJettyLog"/>
+ </logger>
- <logger name="log4j.logger.org.eclipse.jetty" additivity="false" level="info">
- <appender-ref ref="asyncEELFjettyAndNodelog"/>
- </logger>
<root level="INFO">
- <appender-ref ref="asyncEELF" />
- <appender-ref ref="asyncEELFError" />
- <appender-ref ref="asyncEELFjettyAndNodelog" />
+ <appender-ref ref="asyncAudit" />
+ <appender-ref ref="asyncMetrics" />
+ <appender-ref ref="asyncDebug" />
+ <appender-ref ref="asyncError" />
+ <appender-ref ref="asyncJettyLog" />
</root>
-</configuration>
+</configuration> \ No newline at end of file
diff --git a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/node.properties b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/node.properties
index 5de5fc1814..4379dfbf1d 100644
--- a/kubernetes/dmaap/components/dmaap-dr-node/resources/config/node.properties
+++ b/kubernetes/dmaap/components/dmaap-dr-node/resources/config/node.properties
@@ -26,13 +26,13 @@
# URL to retrieve dynamic configuration
#
#ProvisioningURL: ${DRTR_PROV_INTURL}
-ProvisioningURL=https://{{.Values.config.dmaapDrProv.name}}:{{.Values.config.dmaapDrProv.externalPort2}}/internal/prov
+ProvisioningURL=https://{{ .Values.global.dmaapDrProvName }}:{{ .Values.global.dmaapDrProvExtPort2 }}/internal/prov
#
# URL to upload PUB/DEL/EXP logs
#
#LogUploadURL: ${DRTR_LOG_URL}
-LogUploadURL=https://{{.Values.config.dmaapDrProv.name}}:{{.Values.config.dmaapDrProv.externalPort2}}/internal/logs
+LogUploadURL=https://{{ .Values.global.dmaapDrProvName }}:{{ .Values.global.dmaapDrProvExtPort2 }}/internal/logs
#
# The port number for http as seen within the server