diff options
Diffstat (limited to 'src/main/docker/include/etc/confluent')
8 files changed, 256 insertions, 0 deletions
diff --git a/src/main/docker/include/etc/confluent/docker/configure b/src/main/docker/include/etc/confluent/docker/configure new file mode 100644 index 0000000..74e1c11 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/configure @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. /etc/confluent/docker/bash-config + +dub ensure ZOOKEEPER_CLIENT_PORT + +dub path /etc/kafka/ writable + +# myid is required for clusters +if [[ -n "${ZOOKEEPER_SERVERS-}" ]] +then + dub ensure ZOOKEEPER_SERVER_ID + export ZOOKEEPER_INIT_LIMIT=${ZOOKEEPER_INIT_LIMIT:-"10"} + export ZOOKEEPER_SYNC_LIMIT=${ZOOKEEPER_SYNC_LIMIT:-"5"} +fi + +if [[ -n "${ZOOKEEPER_SERVER_ID-}" ]] +then + dub template "/etc/confluent/docker/myid.template" "/var/lib/${COMPONENT}/data/myid" +fi + +if [[ -n "${KAFKA_JMX_OPTS-}" ]] +then + if [[ ! $KAFKA_JMX_OPTS == *"com.sun.management.jmxremote.rmi.port"* ]] + then + echo "KAFKA_JMX_OPTS should contain 'com.sun.management.jmxremote.rmi.port' property. It is required for accessing the JMX metrics externally." + fi +fi + +dub template "/etc/confluent/docker/${COMPONENT}.properties.template" "/etc/kafka/${COMPONENT}.properties" +dub template "/etc/confluent/docker/log4j.properties.template" "/etc/kafka/log4j.properties" +dub template "/etc/confluent/docker/tools-log4j.properties.template" "/etc/kafka/tools-log4j.properties" + + +ZK_REPLICAS=${ZOOKEEPER_REPLICAS:-1} +HOST=$(hostname -s) +DOMAIN=$(hostname -d) +ZK_SERVER_PORT=${ZOOKEEPER_SERVER_PORT:-2888} +ZK_ELECTION_PORT=${ZOOKEEPER_ELECTION_PORT:-3888} + +function print_servers() { + if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then + NAME=${BASH_REMATCH[1]} + for (( i=1; i<=$ZK_REPLICAS; i++ )) + do + echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT" + done + + fi +} + + +if [ $ZK_REPLICAS -gt 1 ]; then +print_servers >> /etc/kafka/${COMPONENT}.properties +fi + +if [ -d /var/lib/zookeeper/data/version-2 ]; then + echo "API Keys already loaded"; + else + cp -var /tmp/zookeeper/gerrit/oom-topics/data-zookeeper/* /var/lib/zookeeper/data/; + rm -rf /tmp/zookeeper/gerrit; + echo "Copying API Keys completed."; + fi + + diff --git a/src/main/docker/include/etc/confluent/docker/ensure b/src/main/docker/include/etc/confluent/docker/ensure new file mode 100644 index 0000000..0f7b8e6 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/ensure @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. /etc/confluent/docker/bash-config + +echo "===> Check if /var/lib/zookeeper/data is writable ..." +dub path /var/lib/zookeeper/data writable + +echo "===> Check if /var/lib/zookeeper/log is writable ..." +dub path /var/lib/zookeeper/log writable diff --git a/src/main/docker/include/etc/confluent/docker/launch b/src/main/docker/include/etc/confluent/docker/launch new file mode 100644 index 0000000..5cdc97e --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/launch @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property. +if [ -z "$KAFKA_JMX_OPTS" ]; then + export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false " +fi + +# The JMX client needs to be able to connect to java.rmi.server.hostname. +# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container. +# For host n/w, this is the IP that the hostname on the host resolves to. + +# If you have more that one n/w configured, hostname -i gives you all the IPs, +# the default is to pick the first IP (or network). +export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)} + +if [ "$KAFKA_JMX_PORT" ]; then + # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger. + export JMX_PORT=$KAFKA_JMX_PORT + export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT" +fi + + +if [[ -n "${ZOOKEEPER_SERVER_ID-}" ]] +then + echo "===> Printing /var/lib/${COMPONENT}/data/myid " + cat /var/lib/"${COMPONENT}"/data/myid +fi + +echo "===> Launching ${COMPONENT} ... " +exec "${COMPONENT}"-server-start /etc/kafka/"${COMPONENT}".properties diff --git a/src/main/docker/include/etc/confluent/docker/log4j.properties.template b/src/main/docker/include/etc/confluent/docker/log4j.properties.template new file mode 100644 index 0000000..972bab3 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/log4j.properties.template @@ -0,0 +1,13 @@ + +log4j.rootLogger={{ env["ZOOKEEPER_LOG4J_ROOT_LOGLEVEL"] | default('INFO') }}, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +{% if env['ZOOKEEPER_LOG4J_LOGGERS'] %} +{% set loggers = parse_log4j_loggers(env['ZOOKEEPER_LOG4J_LOGGERS']) %} +{% for logger,loglevel in loggers.iteritems() %} +log4j.logger.{{logger}}={{loglevel}}, stdout +{% endfor %} +{% endif %} diff --git a/src/main/docker/include/etc/confluent/docker/myid.template b/src/main/docker/include/etc/confluent/docker/myid.template new file mode 100644 index 0000000..d8c94b0 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/myid.template @@ -0,0 +1 @@ +{{env["ZOOKEEPER_SERVER_ID"]}} diff --git a/src/main/docker/include/etc/confluent/docker/run b/src/main/docker/include/etc/confluent/docker/run new file mode 100644 index 0000000..7d311e8 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/run @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Copyright 2016 Confluent Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. /etc/confluent/docker/bash-config + +echo "===> ENV Variables ..." +show_env + +echo "===> User" +id + +echo "===> Configuring ..." +/etc/confluent/docker/configure + +echo "===> Running preflight checks ... " +/etc/confluent/docker/ensure + +echo "===> Launching ... " +exec /etc/confluent/docker/launch diff --git a/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template new file mode 100644 index 0000000..b8ad39c --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template @@ -0,0 +1,7 @@ + +log4j.rootLogger={{ env["ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL"] | default('WARN') }}, stderr + +log4j.appender.stderr=org.apache.log4j.ConsoleAppender +log4j.appender.stderr.layout=org.apache.log4j.PatternLayout +log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n +log4j.appender.stderr.Target=System.err diff --git a/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template b/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template new file mode 100644 index 0000000..e87e6b6 --- /dev/null +++ b/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template @@ -0,0 +1,57 @@ + +clientPort={{ env['ZOOKEEPER_CLIENT_PORT'] }} +dataDir=/var/lib/zookeeper/data +dataLogDir=/var/lib/zookeeper/data + +{# optional properties #} +{% set other_props = { + 'ZOOKEEPER_TICK_TIME': 'tickTime', + 'ZOOKEEPER_GLOBAL_OUTSTANDING_LIMIT' : 'globalOutstandingLimit', + 'ZOOKEEPER_PRE_ALLOC_SIZE': 'preAllocSize', + 'ZOOKEEPER_SNAP_COUNT': 'snapCount', + 'ZOOKEEPER_TRACE_FILE': 'traceFile', + 'ZOOKEEPER_MAX_CLIENT_CNXNS' : 'maxClientCnxns', + 'ZOOKEEPER_CLIENT_PORT_ADDRESS' : 'clientPortAddress', + 'ZOOKEEPER_MIN_SESSION_TIMEOUT' : 'minSessionTimeout', + 'ZOOKEEPER_MAX_SESSION_TIMEOUT' : 'maxSessionTimeout', + 'ZOOKEEPER_FSYNC_WARNING_THRESHOLDMS' : 'fsync.warningthresholdms', + 'ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT' : 'autopurge.snapRetainCount', + 'ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL': 'autopurge.purgeInterval', + 'ZOOKEEPER_SYNC_ENABLED': 'syncEnabled', + 'ZOOKEEPER_ELECTION_ALG' : 'electionAlg', + 'ZOOKEEPER_INIT_LIMIT': 'initLimit', + 'ZOOKEEPER_LEADER_SERVES': 'leaderServes', + 'ZOOKEEPER_SYNC_LIMIT' : 'syncLimit', + 'ZOOKEEPER_CNX_TIMEOUT': 'cnxTimeout', + 'ZOOKEEPER_FORCE_SYNC': 'forceSync', + 'ZOOKEEPER_JUTE_MAX_BUFFER': 'jute.maxbuffer', + 'ZOOKEEPER_SKIP_ACL': 'skipACL', + 'ZOOKEEPER_QUORUM_LISTEN_ON_ALL_IPS': 'quorumListenOnAllIPs' + } -%} + +{% for k, property in other_props.iteritems() -%} +{% if env.get(k) != None -%} +{{property}}={{env[k]}} +{% endif -%} +{% endfor -%} + +{% if env['ZOOKEEPER_SERVERS'] %} +{% set servers = env['ZOOKEEPER_SERVERS'].split(';') %} +{% for server in servers %} +server.{{ loop.index }}={{server}} +{% endfor %} +{% endif %} + +{% if env['ZOOKEEPER_GROUPS'] %} +{% set groups = env['ZOOKEEPER_GROUPS'].split(';') %} +{% for group in groups %} +group.{{ loop.index }}={{group}} +{% endfor %} +{% endif %} + +{% if env['ZOOKEEPER_WEIGHTS'] %} +{% set weights = env['ZOOKEEPER_WEIGHTS'].split(';') %} +{% for weight in weights %} +weight.{{ loop.index }}={{weight}} +{% endfor %} +{% endif %} |