Add Initial Code Import

Added initial code for DMaaP Message Router Component

Issue-id: DMAAP-76
Change-Id: Ica6f265ea4f2901cf47191f21b4448514ea7c8d4
Signed-off-by: Varun Gudisena <vg411h@att.com>

5 files changed, 432 insertions, 0 deletions

diff --git a/etc/cambriaApi.properties b/etc/cambriaApi.properties
new file mode 100644
index 0000000..b3a2b68
--- /dev/null
+++ b/etc/cambriaApi.properties
@@ -0,0 +1,138 @@
+###############################################################################
+#  ============LICENSE_START=======================================================
+#  org.onap.dmaap
+#  ================================================================================
+#  Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+#  ================================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  ============LICENSE_END=========================================================
+#
+#  ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#  
+###############################################################################
+###############################################################################
+##
+## Cambria API Server config
+##
+##	- Default values are shown as commented settings.
+##
+
+###############################################################################
+##
+## HTTP service
+##
+##		- 3904 is standard as of 7/29/14.
+##		- At this time, Cambria always binds to 0.0.0.0
+##
+#cambria.service.port=3904
+#tomcat.maxthreads=(tomcat default, which is usually 200)
+
+###############################################################################
+##
+## Broker Type
+##
+##	The Cambria server can run either as a memory-only implementation, meant
+##	for testing, or against Kafka. For a memory-only server, use "memory" for
+##	the broker.type setting.
+##
+#broker.type=kafka
+
+###############################################################################
+##
+## Zookeeper Connection
+##
+##	Both Cambria and Kafka make use of Zookeeper.
+##
+#config.zk.servers=localhost
+#config.zk.root=/fe3c/cambria/config
+
+
+###############################################################################
+##
+## Kafka Connection
+##
+##	Items below are passed through to Kafka's producer and consumer
+##	configurations (after removing "kafka.")
+##
+#kafka.metadata.broker.list=localhost:9092
+#kafka.client.zookeeper=${config.zk.servers}
+
+###############################################################################
+##
+##	Secured Config
+##
+##	Some data stored in the config system is sensitive -- API keys and secrets,
+##	for example. to protect it, we use an encryption layer for this section
+##	of the config.
+##
+## The key is a base64 encode AES key. This must be created/configured for
+## each installation.
+#cambria.secureConfig.key=
+##
+## The initialization vector is a 16 byte value specific to the secured store.
+## This must be created/configured for each installation.
+#cambria.secureConfig.iv=
+
+## Southfield Sandbox
+#cambria.secureConfig.key=b/7ouTn9FfEw2PQwL0ov/Q==
+#cambria.secureConfig.iv=wR9xP5k5vbz/xD0LmtqQLw==
+
+cambria.secureConfig.key[pc569h]=YT3XPyxEmKCTLI2NK+Sjbw==
+cambria.secureConfig.iv[pc569h]=rMm2jhR3yVnU+u2V9Ugu3Q==
+
+
+###############################################################################
+##
+## Consumer Caching
+##
+##	Kafka expects live connections from the consumer to the broker, which
+##	obviously doesn't work over connectionless HTTP requests. The Cambria
+##	server proxies HTTP requests into Kafka consumer sessions that are kept
+##	around for later re-use. Not doing so is costly for setup per request,
+##	which would substantially impact a high volume consumer's performance.
+##
+##	This complicates Cambria server failover, because we often need server
+##	A to close its connection before server B brings up the replacement.    
+##
+
+## The consumer cache is normally enabled.
+#cambria.consumer.cache.enabled=true
+
+## Cached consumers are cleaned up after a period of disuse. The server inspects
+## consumers every sweepFreqSeconds and will clean up any connections that are
+## dormant for touchFreqMs.
+#cambria.consumer.cache.sweepFreqSeconds=15
+#cambria.consumer.cache.touchFreqMs=120000
+
+## The cache is managed through ZK. The default value for the ZK connection
+## string is the same as config.zk.servers.
+#cambria.consumer.cache.zkConnect=${config.zk.servers}
+
+##
+## Shared cache information is associated with this node's name. The default
+## name is the hostname plus the HTTP service port this host runs on. (The
+## hostname is determined via InetAddress.getLocalHost ().getCanonicalHostName(),
+## which is not always adequate.) You can set this value explicitly here.
+##
+#cambria.api.node.identifier=<use-something-unique-to-this-instance>
+
+###############################################################################
+##
+## Metrics Reporting
+##
+##	This server can report its metrics periodically on a topic.
+##
+#metrics.send.cambria.enabled=true
+#metrics.send.cambria.baseUrl=localhost
+#metrics.send.cambria.topic=cambria.apinode.metrics
+#metrics.send.cambria.sendEverySeconds=60
+
diff --git a/etc/cambriaApi_template.properties b/etc/cambriaApi_template.properties
new file mode 100644
index 0000000..59ed815
--- /dev/null
+++ b/etc/cambriaApi_template.properties
@@ -0,0 +1,130 @@
+###############################################################################
+#  ============LICENSE_START=======================================================
+#  org.onap.dmaap
+#  ================================================================================
+#  Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+#  ================================================================================
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#        http://www.apache.org/licenses/LICENSE-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#  ============LICENSE_END=========================================================
+#
+#  ECOMP is a trademark and service mark of AT&T Intellectual Property.
+#  
+###############################################################################
+###############################################################################
+##
+## Cambria API Server config
+##
+##	- Default values are shown as commented settings.
+##
+
+
+###############################################################################
+##
+## HTTP service
+##
+##		- 3904 is standard as of 7/29/14.
+##		- At this time, Cambria always binds to 0.0.0.0
+##
+cambria.service.port=${CAMBRIA_SERVICE_PORT}
+
+###############################################################################
+##
+## Broker Type
+##
+##	The Cambria server can run either as a memory-only implementation, meant
+##	for testing, or against Kafka. For a memory-only server, use "memory" for
+##	the broker.type setting.
+##
+broker.type=${CAMBRIA_BROKER_TYPE}
+
+###############################################################################
+##
+## Zookeeper Connection
+##
+##	Both Cambria and Kafka make use of Zookeeper.
+#
+config.zk.servers=${CAMBRIA_ZOOKEEPER_NODES}
+config.zk.root=/fe3c/cambria/config
+
+
+###############################################################################
+##
+## Kafka Connection
+##
+##	Items below are passed through to Kafka's producer and consumer
+##	configurations (after removing "kafka.")
+##
+kafka.metadata.broker.list=${KAFKA_BROKER_LIST}
+kafka.client.zookeeper=${CAMBRIA_ZOOKEEPER_NODES}
+
+###############################################################################
+##
+##	Secured Config
+##
+##	Some data stored in the config system is sensitive -- API keys and secrets,
+##	for example. to protect it, we use an encryption layer for this section
+##	of the config.
+##
+## The key is a base64 encode AES key. This must be created/configured for
+## each installation.
+#cambria.secureConfig.key=
+##
+## The initialization vector is a 16 byte value specific to the secured store.
+## This must be created/configured for each installation.
+#cambria.secureConfig.iv=
+
+###############################################################################
+##
+## Consumer Caching
+##
+##	Kafka expects live connections from the consumer to the broker, which
+##	obviously doesn't work over connectionless HTTP requests. The Cambria
+##	server proxies HTTP requests into Kafka consumer sessions that are kept
+##	around for later re-use. Not doing so is costly for setup per request,
+##	which would substantially impact a high volume consumer's performance.
+##
+##	This complicates Cambria server failover, because we often need server
+##	A to close its connection before server B brings up the replacement.    
+##
+
+## The consumer cache is normally enabled.
+cambria.consumer.cache.enabled=true
+
+## Cached consumers are cleaned up after a period of disuse. The server inspects
+## consumers every sweepFreqSeconds and will clean up any connections that are
+## dormant for touchFreqMs.
+cambria.consumer.cache.sweepFreqSeconds=15
+cambria.consumer.cache.touchFreqMs=120000
+
+## The cache is managed through ZK. The default value for the ZK connection
+## string is the same as config.zk.servers.
+cambria.consumer.cache.zkConnect=${CAMBRIA_ZOOKEEPER_NODES}
+
+##
+## Shared cache information is associated with this node's name. The default
+## name is the hostname plus the HTTP service port this host runs on. (The
+## hostname is determined via InetAddress.getLocalHost ().getCanonicalHostName(),
+## which is not always adequate.) You can set this value explicitly here.
+##
+#cambria.api.node.identifier=<use-something-unique-to-this-instance>
+
+###############################################################################
+##
+## Metrics Reporting
+##
+##	This server can report its metrics periodically on a topic.
+##
+metrics.send.cambria.enabled=true
+metrics.send.cambria.baseUrl=localhost:${CAMBRIA_SERVICE_PORT}
+metrics.send.cambria.topic=cambria.apinode.metrics
+metrics.send.cambria.sendEverySeconds=60
+
diff --git a/etc/log4j.xml b/etc/log4j.xml
new file mode 100644
index 0000000..5a15348
--- /dev/null
+++ b/etc/log4j.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+     ============LICENSE_START=======================================================
+     org.onap.dmaap
+     ================================================================================
+     Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+     ================================================================================
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+           http://www.apache.org/licenses/LICENSE-2.0
+     
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+     ============LICENSE_END=========================================================
+   
+     ECOMP is a trademark and service mark of AT&T Intellectual Property.
+     
+ -->
+
+<!DOCTYPE log4j:configuration PUBLIC
+  "-//APACHE//DTD LOG4J 1.2//EN" "http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/xml/doc-files/log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" debug="false">
+
+	<!-- available for console output, not generally used in deployment -->
+	<appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+		<param name="threshold" value="INFO" />
+		<layout class="org.apache.log4j.EnhancedPatternLayout">
+			<param name="ConversionPattern" value="[%d{ISO8601}{GMT+0} GMT][%-10t][%-5p]%m%n" />
+		</layout>
+	</appender>
+
+	<appender name="FILE" class="org.apache.log4j.RollingFileAppender">
+		<param name="threshold" value="INFO" />
+		<param name="File" value="./logs/cambria.log" />	<!-- use local dir by default; prod setup can overwrite -->
+		<param name="MaxFileSize" value="128MB"/>
+		<param name="MaxBackupIndex" value="10"/>
+		<layout class="org.apache.log4j.EnhancedPatternLayout">
+			<param name="ConversionPattern" value="[%d{ISO8601}{GMT+0} GMT][%-10t][%-5p]%m%n" />
+		</layout>
+	</appender>
+
+	<appender name="ECOMP_ERROR" class="org.apache.log4j.RollingFileAppender">
+		<param name="threshold" value="INFO" />
+		<param name="File" value="./logs/error.log" />	<!-- use local dir by default; prod setup can overwrite -->
+		<param name="MaxFileSize" value="128MB"/>
+		<param name="MaxBackupIndex" value="10"/>
+		<layout class="org.apache.log4j.EnhancedPatternLayout">
+			<param name="ConversionPattern" value="%d{yyyy-MM-dd'T'HH:mm:ss}{GMT+0}+00:00|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{instanceUuid}|%p|%X{severity}|%X{serverIpAddress}|%X{server}|%X{ipAddress}|%X{className}|%X{timer}|%m%n" />
+		</layout>
+	</appender>
+	<!-- the other 3 ECOMP logs are omitted for this release -->
+
+	<root>
+		<level value="INFO" />
+		<appender-ref ref="FILE" />
+		<appender-ref ref="ECOMP_ERROR" />
+		<appender-ref ref="CONSOLE" />
+	</root>
+
+</log4j:configuration>
diff --git a/etc/log4j_template.xml b/etc/log4j_template.xml
new file mode 100644
index 0000000..808a1bc
--- /dev/null
+++ b/etc/log4j_template.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+     ============LICENSE_START=======================================================
+     org.onap.dmaap
+     ================================================================================
+     Copyright © 2017 AT&T Intellectual Property. All rights reserved.
+     ================================================================================
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+           http://www.apache.org/licenses/LICENSE-2.0
+     
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+     ============LICENSE_END=========================================================
+   
+     ECOMP is a trademark and service mark of AT&T Intellectual Property.
+     
+ -->
+
+<!DOCTYPE log4j:configuration PUBLIC
+  "-//APACHE//DTD LOG4J 1.2//EN" "http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/xml/doc-files/log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/" debug="false">
+
+	<appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+		<param name="threshold" value="INFO" />
+		<layout class="org.apache.log4j.EnhancedPatternLayout">
+			<param name="ConversionPattern" value="[%d{ISO8601}{GMT+0} GMT][%-10t][%-5p][%X{serverIp}]%m%n" />
+		</layout>
+	</appender>
+
+	<appender name="FILE" class="org.apache.log4j.RollingFileAppender">
+		<param name="threshold" value="${CAMBRIA_LOG_THRESHOLD}" />
+		<param name="File" value="${CAMBRIA_LOG_DIR}/cambria.log" />
+		<param name="MaxFileSize" value="128MB"/>
+		<param name="MaxBackupIndex" value="10"/>
+		<layout class="org.apache.log4j.EnhancedPatternLayout">
+			<param name="ConversionPattern" value="%d{yyyy-MM-dd'T'HH:mm:ss}{GMT+0}+00:00|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{instanceUuid}|%p|%X{severity}|%X{serverIpAddress}|%X{server}|%X{ipAddress}|%X{className}|%X{timer}|%m%n" />
+		</layout>
+	</appender>
+
+	<!-- The ECOMP error.log log -->
+	<appender name="ECOMP_ERROR" class="org.apache.log4j.RollingFileAppender">
+		<param name="threshold" value="INFO" />
+		<param name="File" value="${CAMBRIA_LOG_DIR}/error.log" />
+		<param name="MaxFileSize" value="128MB"/>
+		<param name="MaxBackupIndex" value="10"/>
+		<layout class="org.apache.log4j.EnhancedPatternLayout">
+			<param name="ConversionPattern" value="%d{yyyy-MM-dd'T'HH:mm:ss}{GMT+0}+00:00|%X{requestId}|%X{serviceInstanceId}|%-10t|%X{serverName}|%X{serviceName}|%X{instanceUuid}|%p|%X{severity}|%X{serverIpAddress}|%X{server}|%X{ipAddress}|%X{className}|%X{timer}|%m%n" />
+		</layout>
+	</appender>
+	<!-- the other 3 ECOMP logs are omitted for this release -->
+
+	<root>
+		<level value="${CAMBRIA_LOG_THRESHOLD}" />
+		<appender-ref ref="FILE" />
+	</root>
+
+</log4j:configuration>
diff --git a/etc/logstash_cambria_template.conf b/etc/logstash_cambria_template.conf
new file mode 100644
index 0000000..2ddb7d5
--- /dev/null
+++ b/etc/logstash_cambria_template.conf
@@ -0,0 +1,36 @@
+input {
+        file {
+                path => "${CAMBRIA_SERVER_LOG}"
+                codec => multiline {
+                        pattern => "^\[%{YEAR}-%{MONTH}-%{MONTHDAY}%{SPACE}%{HOUR}:%{MINUTE}:%{SECOND}\]"
+                        negate => "true"
+                        what => "previous"
+                }
+                sincedb_path => "/opt/app/logstash/conf_stage/.sincedb_cambria"
+                start_position => "beginning"
+                type => "cambria"
+        }
+}
+
+filter {
+	if [type] == "cambria" {
+	        grok {
+	                match => ["message", "\[(?<date>%{YEAR}-%{MONTH}-%{MONTHDAY}%{SPACE}%{HOUR}:%{MINUTE}:%{SECOND})\]\[%{DATA:logLevel}\]\[%{DATA:thread}\]\[%{DATA:class}\]\[%{DATA:id}\]%{GREEDYDATA:message}"]
+	        }
+	
+	        date {
+	                match => ["date", "YYYY-MMM-DD HH:mm:ss,SSS"]
+	        }
+	}
+}
+
+output {
+	if [type] == "cambria" {
+            elasticsearch {
+                    cluster => "2020SA"
+                    host => "${ELASTICSEARCH_NODES}"
+                    index => "cambria-%{+YYYY.MM.dd}"
+            }
+    }
+}
+