DMAAP-MR - Merge MR repos

Issue-ID: DMAAP-1582
Signed-off-by: david.mcweeney <david.mcweeney@est.tech>
Change-Id: I727cffc687a7ea79abbd858affafc823c0f96f0b

5 files changed, 426 insertions, 0 deletions

diff --git a/etc/cambriaApi.properties b/etc/cambriaApi.properties
new file mode 100644
index 0000000..b3a2b68
--- /dev/null
+++ b/etc/cambriaApi.properties
@@ -0,0 +1,138 @@
+###############################################################################
+#  ============LICENSE_START=======================================================
+#  org.onap.dmaap
+#  ================================================================================
+#  Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+#  ================================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  ============LICENSE_END=========================================================
+#
+#  ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#  
+###############################################################################
+###############################################################################
+##
+## Cambria API Server config
+##
+##	- Default values are shown as commented settings.
+##
+
+###############################################################################
+##
+## HTTP service
+##
+##		- 3904 is standard as of 7/29/14.
+##		- At this time, Cambria always binds to 0.0.0.0
+##
+#cambria.service.port=3904
+#tomcat.maxthreads=(tomcat default, which is usually 200)
+
+###############################################################################
+##
+## Broker Type
+##
+##	The Cambria server can run either as a memory-only implementation, meant
+##	for testing, or against Kafka. For a memory-only server, use "memory" for
+##	the broker.type setting.
+##
+#broker.type=kafka
+
+###############################################################################
+##
+## Zookeeper Connection
+##
+##	Both Cambria and Kafka make use of Zookeeper.
+##
+#config.zk.servers=localhost
+#config.zk.root=/fe3c/cambria/config
+
+
+###############################################################################
+##
+## Kafka Connection
+##
+##	Items below are passed through to Kafka's producer and consumer
+##	configurations (after removing "kafka.")
+##
+#kafka.metadata.broker.list=localhost:9092
+#kafka.client.zookeeper=${config.zk.servers}
+
+###############################################################################
+##
+##	Secured Config
+##
+##	Some data stored in the config system is sensitive -- API keys and secrets,
+##	for example. to protect it, we use an encryption layer for this section
+##	of the config.
+##
+## The key is a base64 encode AES key. This must be created/configured for
+## each installation.
+#cambria.secureConfig.key=
+##
+## The initialization vector is a 16 byte value specific to the secured store.
+## This must be created/configured for each installation.
+#cambria.secureConfig.iv=
+
+## Southfield Sandbox
+#cambria.secureConfig.key=b/7ouTn9FfEw2PQwL0ov/Q==
+#cambria.secureConfig.iv=wR9xP5k5vbz/xD0LmtqQLw==
+
+cambria.secureConfig.key[pc569h]=YT3XPyxEmKCTLI2NK+Sjbw==
+cambria.secureConfig.iv[pc569h]=rMm2jhR3yVnU+u2V9Ugu3Q==
+
+
+###############################################################################
+##
+## Consumer Caching
+##
+##	Kafka expects live connections from the consumer to the broker, which
+##	obviously doesn't work over connectionless HTTP requests. The Cambria
+##	server proxies HTTP requests into Kafka consumer sessions that are kept
+##	around for later re-use. Not doing so is costly for setup per request,
+##	which would substantially impact a high volume consumer's performance.
+##
+##	This complicates Cambria server failover, because we often need server
+##	A to close its connection before server B brings up the replacement.    
+##
+
+## The consumer cache is normally enabled.
+#cambria.consumer.cache.enabled=true
+
+## Cached consumers are cleaned up after a period of disuse. The server inspects
+## consumers every sweepFreqSeconds and will clean up any connections that are
+## dormant for touchFreqMs.
+#cambria.consumer.cache.sweepFreqSeconds=15
+#cambria.consumer.cache.touchFreqMs=120000
+
+## The cache is managed through ZK. The default value for the ZK connection
+## string is the same as config.zk.servers.
+#cambria.consumer.cache.zkConnect=${config.zk.servers}
+
+##
+## Shared cache information is associated with this node's name. The default
+## name is the hostname plus the HTTP service port this host runs on. (The
+## hostname is determined via InetAddress.getLocalHost ().getCanonicalHostName(),
+## which is not always adequate.) You can set this value explicitly here.
+##
+#cambria.api.node.identifier=<use-something-unique-to-this-instance>
+
+###############################################################################
+##
+## Metrics Reporting
+##
+##	This server can report its metrics periodically on a topic.
+##
+#metrics.send.cambria.enabled=true
+#metrics.send.cambria.baseUrl=localhost
+#metrics.send.cambria.topic=cambria.apinode.metrics
+#metrics.send.cambria.sendEverySeconds=60
+
diff --git a/etc/cambriaApi_template.properties b/etc/cambriaApi_template.properties
new file mode 100644
index 0000000..59ed815
--- /dev/null
+++ b/etc/cambriaApi_template.properties
@@ -0,0 +1,130 @@
+###############################################################################
+#  ============LICENSE_START=======================================================
+#  org.onap.dmaap
+#  ================================================================================
+#  Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+#  ================================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  ============LICENSE_END=========================================================
+#
+#  ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#  
+###############################################################################
+###############################################################################
+##
+## Cambria API Server config
+##
+##	- Default values are shown as commented settings.
+##
+
+
+###############################################################################
+##
+## HTTP service
+##
+##		- 3904 is standard as of 7/29/14.
+##		- At this time, Cambria always binds to 0.0.0.0
+##
+cambria.service.port=${CAMBRIA_SERVICE_PORT}
+
+###############################################################################
+##
+## Broker Type
+##
+##	The Cambria server can run either as a memory-only implementation, meant
+##	for testing, or against Kafka. For a memory-only server, use "memory" for
+##	the broker.type setting.
+##
+broker.type=${CAMBRIA_BROKER_TYPE}
+
+###############################################################################
+##
+## Zookeeper Connection
+##
+##	Both Cambria and Kafka make use of Zookeeper.
+#
+config.zk.servers=${CAMBRIA_ZOOKEEPER_NODES}
+config.zk.root=/fe3c/cambria/config
+
+
+###############################################################################
+##
+## Kafka Connection
+##
+##	Items below are passed through to Kafka's producer and consumer
+##	configurations (after removing "kafka.")
+##
+kafka.metadata.broker.list=${KAFKA_BROKER_LIST}
+kafka.client.zookeeper=${CAMBRIA_ZOOKEEPER_NODES}
+
+###############################################################################
+##
+##	Secured Config
+##
+##	Some data stored in the config system is sensitive -- API keys and secrets,
+##	for example. to protect it, we use an encryption layer for this section
+##	of the config.
+##
+## The key is a base64 encode AES key. This must be created/configured for
+## each installation.
+#cambria.secureConfig.key=
+##
+## The initialization vector is a 16 byte value specific to the secured store.
+## This must be created/configured for each installation.
+#cambria.secureConfig.iv=
+
+###############################################################################
+##
+## Consumer Caching
+##
+##	Kafka expects live connections from the consumer to the broker, which
+##	obviously doesn't work over connectionless HTTP requests. The Cambria
+##	server proxies HTTP requests into Kafka consumer sessions that are kept
+##	around for later re-use. Not doing so is costly for setup per request,
+##	which would substantially impact a high volume consumer's performance.
+##
+##	This complicates Cambria server failover, because we often need server
+##	A to close its connection before server B brings up the replacement.    
+##
+
+## The consumer cache is normally enabled.
+cambria.consumer.cache.enabled=true
+
+## Cached consumers are cleaned up after a period of disuse. The server inspects
+## consumers every sweepFreqSeconds and will clean up any connections that are
+## dormant for touchFreqMs.
+cambria.consumer.cache.sweepFreqSeconds=15
+cambria.consumer.cache.touchFreqMs=120000
+
+## The cache is managed through ZK. The default value for the ZK connection
+## string is the same as config.zk.servers.
+cambria.consumer.cache.zkConnect=${CAMBRIA_ZOOKEEPER_NODES}
+
+##
+## Shared cache information is associated with this node's name. The default
+## name is the hostname plus the HTTP service port this host runs on. (The
+## hostname is determined via InetAddress.getLocalHost ().getCanonicalHostName(),
+## which is not always adequate.) You can set this value explicitly here.
+##
+#cambria.api.node.identifier=<use-something-unique-to-this-instance>
+
+###############################################################################
+##
+## Metrics Reporting
+##
+##	This server can report its metrics periodically on a topic.
+##
+metrics.send.cambria.enabled=true
+metrics.send.cambria.baseUrl=localhost:${CAMBRIA_SERVICE_PORT}
+metrics.send.cambria.topic=cambria.apinode.metrics
+metrics.send.cambria.sendEverySeconds=60
+
diff --git a/etc/log4j2.xml b/etc/log4j2.xml
new file mode 100644
index 0000000..0918f35
--- /dev/null
+++ b/etc/log4j2.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ============LICENSE_START=======================================================
+  Copyright (C) 2021 Nordix Foundation.
+  ================================================================================
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  SPDX-License-Identifier: Apache-2.0
+  ============LICENSE_END=========================================================
+  -->
+
+<Configuration status="WARN">
+    <Appenders>
+        <!-- Console Appender -->
+        <Console name="STDOUT" target="SYSTEM_OUT">
+            <PatternLayout pattern="[%d{ISO8601}{GMT+0} GMT][%-10t][%-5p]%m%n"/>
+        </Console>
+
+        <!-- Rolling File Appender -->
+        <RollingFile name="rollingFile">
+            <FileName>./logs/cambria.log</FileName>
+            <FilePattern>./logs/${date:yyyy-MM}/cambria-%d{yyyy-MM-dd}-%i.log</FilePattern>
+            <PatternLayout>
+                <Pattern>[%d{ISO8601}{GMT+0} GMT][%-10t][%-5p]%m%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="128 MB"/>
+            </Policies>
+            <DefaultRolloverStrategy max="10"/>
+        </RollingFile>
+
+        <!-- Rolling File Appender -->
+        <RollingFile name="rollingFile_ECOMP_ERROR">
+            <FileName>./logs/error.log</FileName>
+            <FilePattern>./logs/${date:yyyy-MM}/error-%d{yyyy-MM-dd}-%i.log</FilePattern>
+            <PatternLayout>
+                <Pattern>%d{yyyy-MM-dd'T'HH:mm:ss}{GMT+0}+00:00|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{instanceUuid}|%p|%X{severity}|%X{serverIpAddress}|%X{server}|%X{ipAddress}|%X{className}|%X{timer}|%m%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="128 MB"/>
+            </Policies>
+            <DefaultRolloverStrategy max="10"/>
+        </RollingFile>
+    </Appenders>
+    <Loggers>
+        <Root level="info">
+            <AppenderRef ref="STDOUT"/>
+            <AppenderRef ref="rollingFile"/>
+            <AppenderRef ref="rollingFile_ECOMP_ERROR"/>
+        </Root>
+    </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/etc/log4j2_template.xml b/etc/log4j2_template.xml
new file mode 100644
index 0000000..1c70479
--- /dev/null
+++ b/etc/log4j2_template.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ============LICENSE_START=======================================================
+  Copyright (C) 2021 Nordix Foundation.
+  ================================================================================
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+  SPDX-License-Identifier: Apache-2.0
+  ============LICENSE_END=========================================================
+  -->
+
+<Configuration status="WARN">
+    <Appenders>
+        <!-- Console Appender -->
+        <Console name="STDOUT" target="SYSTEM_OUT">
+            <PatternLayout pattern="[%d{ISO8601}{GMT+0} GMT][%-10t][%-5p][%X{serverIp}]%m%n"/>
+        </Console>
+
+        <!-- Rolling File Appender -->
+        <RollingFile name="rollingFile">
+            <FileName>./${CAMBRIA_LOG_DIR}/cambria.log</FileName>
+            <FilePattern>./${CAMBRIA_LOG_DIR}/${date:yyyy-MM}/cambria-%d{yyyy-MM-dd}-%i.log</FilePattern>
+            <PatternLayout>
+                <Pattern>%d{yyyy-MM-dd'T'HH:mm:ss}{GMT+0}+00:00|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{instanceUuid}|%p|%X{severity}|%X{serverIpAddress}|%X{server}|%X{ipAddress}|%X{className}|%X{timer}|%m%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="128 MB"/>
+            </Policies>
+            <DefaultRolloverStrategy max="10"/>
+        </RollingFile>
+
+        <!-- Rolling File Appender -->
+        <RollingFile name="rollingFile_ECOMP_ERROR">
+            <FileName>./${CAMBRIA_LOG_DIR}/error.log</FileName>
+            <FilePattern>./${CAMBRIA_LOG_DIR}/${date:yyyy-MM}/error-%d{yyyy-MM-dd}-%i.log</FilePattern>
+            <PatternLayout>
+                <Pattern>%d{yyyy-MM-dd'T'HH:mm:ss}{GMT+0}+00:00|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{instanceUuid}|%p|%X{severity}|%X{serverIpAddress}|%X{server}|%X{ipAddress}|%X{className}|%X{timer}|%m%n</Pattern>
+            </PatternLayout>
+            <Policies>
+                <SizeBasedTriggeringPolicy size="128 MB"/>
+            </Policies>
+            <DefaultRolloverStrategy max="10"/>
+        </RollingFile>
+    </Appenders>
+    <Loggers>
+        <Root level="${CAMBRIA_LOG_THRESHOLD}">
+            <AppenderRef ref="rollingFile"/>
+        </Root>
+    </Loggers>
+</Configuration>
\ No newline at end of file
diff --git a/etc/logstash_cambria_template.conf b/etc/logstash_cambria_template.conf
new file mode 100644
index 0000000..2ddb7d5
--- /dev/null
+++ b/etc/logstash_cambria_template.conf
@@ -0,0 +1,36 @@
+input {
+        file {
+                path => "${CAMBRIA_SERVER_LOG}"
+                codec => multiline {
+                        pattern => "^\[%{YEAR}-%{MONTH}-%{MONTHDAY}%{SPACE}%{HOUR}:%{MINUTE}:%{SECOND}\]"
+                        negate => "true"
+                        what => "previous"
+                }
+                sincedb_path => "/opt/app/logstash/conf_stage/.sincedb_cambria"
+                start_position => "beginning"
+                type => "cambria"
+        }
+}
+
+filter {
+	if [type] == "cambria" {
+	        grok {
+	                match => ["message", "\[(?<date>%{YEAR}-%{MONTH}-%{MONTHDAY}%{SPACE}%{HOUR}:%{MINUTE}:%{SECOND})\]\[%{DATA:logLevel}\]\[%{DATA:thread}\]\[%{DATA:class}\]\[%{DATA:id}\]%{GREEDYDATA:message}"]
+	        }
+	
+	        date {
+	                match => ["date", "YYYY-MMM-DD HH:mm:ss,SSS"]
+	        }
+	}
+}
+
+output {
+	if [type] == "cambria" {
+            elasticsearch {
+                    cluster => "2020SA"
+                    host => "${ELASTICSEARCH_NODES}"
+                    index => "cambria-%{+YYYY.MM.dd}"
+            }
+    }
+}
+