diff options
Diffstat (limited to 'cdap3vm/config/hadoop-cluster-config-template/kafka')
8 files changed, 361 insertions, 0 deletions
diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties new file mode 100644 index 0000000..83847de --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/consumer.properties @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.consumer.ConsumerConfig for more details + +# Zookeeper connection string +# comma separated host:port pairs, each corresponding to a zk +# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002" +zookeeper.connect=127.0.0.1:2181 + +# timeout in ms for connecting to zookeeper +zookeeper.connection.timeout.ms=6000 + +#consumer group id +group.id=test-consumer-group + +#consumer timeout +#consumer.timeout.ms=5000 diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh b/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh new file mode 100644 index 0000000..cbfc937 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/kafka-env.sh @@ -0,0 +1,19 @@ + +#!/bin/bash + +# Set KAFKA specific environment variables here. + +# The java implementation to use. +export JAVA_HOME=/usr/jdk64/jdk1.8.0_40 +export PATH=$PATH:$JAVA_HOME/bin +export PID_DIR=/var/run/kafka +export LOG_DIR=/var/log/kafka +export KAFKA_KERBEROS_PARAMS= +# Add kafka sink to classpath and related depenencies +if [ -e "/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar" ]; then + export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar + export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/* +fi +if [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then +. /etc/kafka/conf/kafka-ranger-env.sh +fi diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties new file mode 100644 index 0000000..2cba04d --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/log4j.properties @@ -0,0 +1,84 @@ + +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# +# +kafka.logs.dir=logs + +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log +log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +#log4j.logger.kafka.perf=DEBUG, kafkaAppender +#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=INFO, kafkaAppender +log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=WARN, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender +log4j.additivity.kafka.log.LogCleaner=false + +log4j.logger.state.change.logger=TRACE, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties new file mode 100644 index 0000000..47ae3e2 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/producer.properties @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# see kafka.producer.ProducerConfig for more details + +############################# Producer Basics ############################# + +# list of brokers used for bootstrapping knowledge about the rest of the cluster +# format: host1:port1,host2:port2 ... +metadata.broker.list=localhost:9092 + +# name of the partitioner class for partitioning events; default partition spreads data randomly +#partitioner.class= + +# specifies whether the messages are sent asynchronously (async) or synchronously (sync) +producer.type=sync + +# specify the compression codec for all data generated: none, gzip, snappy, lz4. +# the old config values work as well: 0, 1, 2, 3 for none, gzip, snappy, lz4, respectively +compression.codec=none + +# message encoder +serializer.class=kafka.serializer.DefaultEncoder + +# allow topic level compression +#compressed.topics= + +############################# Async Producer ############################# +# maximum time, in milliseconds, for buffering data on the producer queue +#queue.buffering.max.ms= + +# the maximum size of the blocking queue for buffering on the producer +#queue.buffering.max.messages= + +# Timeout for event enqueue: +# 0: events will be enqueued immediately or dropped if the queue is full +# -ve: enqueue will block indefinitely if the queue is full +# +ve: enqueue will block up to this many milliseconds if the queue is full +#queue.enqueue.timeout.ms= + +# the number of messages batched at the producer +#batch.num.messages= diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties new file mode 100644 index 0000000..7dfb879 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/server.properties @@ -0,0 +1,70 @@ +auto.leader.rebalance.enable=true +kafka.ganglia.metrics.port=8671 +socket.send.buffer.bytes=102400 +message.max.bytes=1000000 +num.network.threads=3 +log.segment.bytes=1073741824 +kafka.ganglia.metrics.host=localhost +leader.imbalance.per.broker.percentage=10 +replica.lag.time.max.ms=10000 +authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer +num.io.threads=8 +offsets.retention.minutes=86400000 +fetch.purgatory.purge.interval.requests=10000 +offsets.topic.compression.codec=0 +replica.lag.max.messages=4000 +num.recovery.threads.per.data.dir=1 +log.dirs=/grid/0/kafka-logs +log.retention.bytes=-1 +producer.purgatory.purge.interval.requests=10000 +log.flush.scheduler.interval.ms=3000 +default.replication.factor=1 +replica.high.watermark.checkpoint.interval.ms=5000 +zookeeper.connect=TODO_REPLACE_HOSTNAME:2181 +controlled.shutdown.retry.backoff.ms=5000 +kafka.timeline.metrics.host=TODO_REPLACE_HOSTNAME +kafka.ganglia.metrics.group=kafka +kafka.timeline.metrics.reporter.sendInterval=5900 +num.partitions=1 +offsets.topic.segment.bytes=104857600 +controller.socket.timeout.ms=30000 +queued.max.requests=500 +broker.id=0 +offset.metadata.max.bytes=4096 +kafka.timeline.metrics.reporter.enabled=true +controlled.shutdown.max.retries=3 +min.insync.replicas=1 +offsets.commit.required.acks=-1 +replica.fetch.wait.max.ms=500 +controlled.shutdown.enable=true +log.roll.hours=168 +log.cleanup.interval.mins=10 +replica.socket.receive.buffer.bytes=65536 +kafka.ganglia.metrics.reporter.enabled=true +offsets.topic.num.partitions=50 +delete.topic.enable=false +offsets.load.buffer.size=5242880 +num.replica.fetchers=1 +socket.request.max.bytes=104857600 +kafka.timeline.metrics.maxRowCacheSize=10000 +kafka.timeline.metrics.port=6188 +compression.type=producer +zookeeper.sync.time.ms=2000 +zookeeper.connection.timeout.ms=6000 +socket.receive.buffer.bytes=102400 +controller.message.queue.size=10 +offsets.commit.timeout.ms=5000 +offsets.topic.replication.factor=3 +offsets.retention.check.interval.ms=600000 +log.flush.interval.ms=3000 +replica.fetch.max.bytes=1048576 +kafka.metrics.reporters=org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter +listeners=PLAINTEXT://TODO_REPLACE_HOSTNAME:6667 +replica.fetch.min.bytes=1 +replica.socket.timeout.ms=30000 +zookeeper.session.timeout.ms=30000 +auto.create.topics.enable=true +log.index.size.max.bytes=10485760 +leader.imbalance.check.interval.seconds=300 +log.index.interval.bytes=4096 +log.retention.hours=168 diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties new file mode 100644 index 0000000..e0bbc13 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/test-log4j.properties @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.kafkaAppender.File=logs/server.log +log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.stateChangeAppender.File=logs/state-change.log +log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.requestAppender.File=logs/kafka-request.log +log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender +log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH +log4j.appender.controllerAppender.File=logs/controller.log +log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n + +# Turn on all our debugging info +#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender +#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender +log4j.logger.kafka.tools=DEBUG, kafkaAppender +log4j.logger.kafka.tools.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender +#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG +log4j.logger.kafka=INFO, kafkaAppender + +log4j.logger.kafka.network.RequestChannel$=TRACE, requestAppender +log4j.additivity.kafka.network.RequestChannel$=false + +#log4j.logger.kafka.network.Processor=TRACE, requestAppender +#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender +#log4j.additivity.kafka.server.KafkaApis=false +log4j.logger.kafka.request.logger=TRACE, requestAppender +log4j.additivity.kafka.request.logger=false + +log4j.logger.kafka.controller=TRACE, controllerAppender +log4j.additivity.kafka.controller=false + +log4j.logger.state.change.logger=TRACE, stateChangeAppender +log4j.additivity.state.change.logger=false diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties new file mode 100644 index 0000000..52f07c9 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/tools-log4j.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +log4j.rootLogger=WARN, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n diff --git a/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties b/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties new file mode 100644 index 0000000..74cbf90 --- /dev/null +++ b/cdap3vm/config/hadoop-cluster-config-template/kafka/zookeeper.properties @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# the directory where the snapshot is stored. +dataDir=/tmp/zookeeper +# the port at which the clients will connect +clientPort=2181 +# disable the per-ip limit on the number of connections since this is a non-production config +maxClientCnxns=0 |