confluent based image

Issue-ID: DMAAP-1339
Change-Id: Ieb942450ea48ca246876cd0fcf02ba59187cabe2
Signed-off-by: su622b <su622b@att.com>

8 files changed, 256 insertions, 0 deletions

diff --git a/src/main/docker/include/etc/confluent/docker/configure b/src/main/docker/include/etc/confluent/docker/configure
new file mode 100644
index 0000000..74e1c11
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/configure
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/confluent/docker/bash-config
+
+dub ensure ZOOKEEPER_CLIENT_PORT
+
+dub path /etc/kafka/ writable
+
+# myid is required for clusters
+if [[ -n "${ZOOKEEPER_SERVERS-}" ]]
+then
+  dub ensure ZOOKEEPER_SERVER_ID
+  export ZOOKEEPER_INIT_LIMIT=${ZOOKEEPER_INIT_LIMIT:-"10"}
+  export ZOOKEEPER_SYNC_LIMIT=${ZOOKEEPER_SYNC_LIMIT:-"5"}
+fi
+
+if [[ -n "${ZOOKEEPER_SERVER_ID-}" ]]
+then
+  dub template "/etc/confluent/docker/myid.template" "/var/lib/${COMPONENT}/data/myid"
+fi
+
+if [[ -n "${KAFKA_JMX_OPTS-}" ]]
+then
+  if [[ ! $KAFKA_JMX_OPTS == *"com.sun.management.jmxremote.rmi.port"*  ]]
+  then
+    echo "KAFKA_JMX_OPTS should contain 'com.sun.management.jmxremote.rmi.port' property. It is required for accessing the JMX metrics externally."
+  fi
+fi
+
+dub template "/etc/confluent/docker/${COMPONENT}.properties.template" "/etc/kafka/${COMPONENT}.properties"
+dub template "/etc/confluent/docker/log4j.properties.template" "/etc/kafka/log4j.properties"
+dub template "/etc/confluent/docker/tools-log4j.properties.template" "/etc/kafka/tools-log4j.properties"
+
+
+ZK_REPLICAS=${ZOOKEEPER_REPLICAS:-1}
+HOST=$(hostname -s)
+DOMAIN=$(hostname -d)
+ZK_SERVER_PORT=${ZOOKEEPER_SERVER_PORT:-2888}
+ZK_ELECTION_PORT=${ZOOKEEPER_ELECTION_PORT:-3888}
+
+function print_servers() {
+   if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
+        NAME=${BASH_REMATCH[1]}
+        for (( i=1; i<=$ZK_REPLICAS; i++ ))
+    do
+        echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT"
+    done
+  
+    fi
+}
+
+
+if [ $ZK_REPLICAS -gt 1 ]; then
+print_servers >> /etc/kafka/${COMPONENT}.properties
+fi
+
+if [ -d /var/lib/zookeeper/data/version-2 ]; then
+          echo "API Keys already loaded";
+          else
+          cp -var /tmp/zookeeper/gerrit/oom-topics/data-zookeeper/* /var/lib/zookeeper/data/;
+          rm -rf /tmp/zookeeper/gerrit;
+          echo "Copying API Keys completed.";
+     fi
+
+
diff --git a/src/main/docker/include/etc/confluent/docker/ensure b/src/main/docker/include/etc/confluent/docker/ensure
new file mode 100644
index 0000000..0f7b8e6
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/ensure
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/confluent/docker/bash-config
+
+echo "===> Check if /var/lib/zookeeper/data is writable ..."
+dub path /var/lib/zookeeper/data writable
+
+echo "===> Check if /var/lib/zookeeper/log is writable ..."
+dub path /var/lib/zookeeper/log writable
diff --git a/src/main/docker/include/etc/confluent/docker/launch b/src/main/docker/include/etc/confluent/docker/launch
new file mode 100644
index 0000000..5cdc97e
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/launch
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property.
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# The JMX client needs to be able to connect to java.rmi.server.hostname.
+# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container.
+# For host n/w, this is the IP that the hostname on the host resolves to.
+
+# If you have more that one n/w configured, hostname -i gives you all the IPs,
+# the default is to pick the first IP (or network).
+export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)}
+
+if [ "$KAFKA_JMX_PORT" ]; then
+  # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger.
+  export JMX_PORT=$KAFKA_JMX_PORT
+  export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT"
+fi
+
+
+if [[ -n "${ZOOKEEPER_SERVER_ID-}" ]]
+then
+  echo "===> Printing /var/lib/${COMPONENT}/data/myid "
+  cat /var/lib/"${COMPONENT}"/data/myid
+fi
+
+echo "===> Launching ${COMPONENT} ... "
+exec "${COMPONENT}"-server-start /etc/kafka/"${COMPONENT}".properties
diff --git a/src/main/docker/include/etc/confluent/docker/log4j.properties.template b/src/main/docker/include/etc/confluent/docker/log4j.properties.template
new file mode 100644
index 0000000..972bab3
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/log4j.properties.template
@@ -0,0 +1,13 @@
+
+log4j.rootLogger={{ env["ZOOKEEPER_LOG4J_ROOT_LOGLEVEL"] | default('INFO') }}, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+{% if env['ZOOKEEPER_LOG4J_LOGGERS'] %}
+{% set loggers = parse_log4j_loggers(env['ZOOKEEPER_LOG4J_LOGGERS']) %}
+{% for logger,loglevel in loggers.iteritems() %}
+log4j.logger.{{logger}}={{loglevel}}, stdout
+{% endfor %}
+{% endif %}
diff --git a/src/main/docker/include/etc/confluent/docker/myid.template b/src/main/docker/include/etc/confluent/docker/myid.template
new file mode 100644
index 0000000..d8c94b0
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/myid.template
@@ -0,0 +1 @@
+{{env["ZOOKEEPER_SERVER_ID"]}}
diff --git a/src/main/docker/include/etc/confluent/docker/run b/src/main/docker/include/etc/confluent/docker/run
new file mode 100644
index 0000000..7d311e8
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/run
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/confluent/docker/bash-config
+
+echo "===> ENV Variables ..."
+show_env
+
+echo "===> User"
+id
+
+echo "===> Configuring ..."
+/etc/confluent/docker/configure
+
+echo "===> Running preflight checks ... "
+/etc/confluent/docker/ensure
+
+echo "===> Launching ... "
+exec /etc/confluent/docker/launch
diff --git a/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template
new file mode 100644
index 0000000..b8ad39c
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template
@@ -0,0 +1,7 @@
+
+log4j.rootLogger={{ env["ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL"] | default('WARN') }}, stderr
+
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n
+log4j.appender.stderr.Target=System.err
diff --git a/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template b/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template
new file mode 100644
index 0000000..e87e6b6
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template
@@ -0,0 +1,57 @@
+
+clientPort={{ env['ZOOKEEPER_CLIENT_PORT'] }}
+dataDir=/var/lib/zookeeper/data
+dataLogDir=/var/lib/zookeeper/data
+
+{# optional properties #}
+{% set other_props = {
+  'ZOOKEEPER_TICK_TIME': 'tickTime',
+  'ZOOKEEPER_GLOBAL_OUTSTANDING_LIMIT' : 'globalOutstandingLimit',
+  'ZOOKEEPER_PRE_ALLOC_SIZE': 'preAllocSize',
+  'ZOOKEEPER_SNAP_COUNT': 'snapCount',
+  'ZOOKEEPER_TRACE_FILE': 'traceFile',
+  'ZOOKEEPER_MAX_CLIENT_CNXNS' : 'maxClientCnxns',
+  'ZOOKEEPER_CLIENT_PORT_ADDRESS' : 'clientPortAddress',
+  'ZOOKEEPER_MIN_SESSION_TIMEOUT' : 'minSessionTimeout',
+  'ZOOKEEPER_MAX_SESSION_TIMEOUT' : 'maxSessionTimeout',
+  'ZOOKEEPER_FSYNC_WARNING_THRESHOLDMS' : 'fsync.warningthresholdms',
+  'ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT' : 'autopurge.snapRetainCount',
+  'ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL': 'autopurge.purgeInterval',
+  'ZOOKEEPER_SYNC_ENABLED': 'syncEnabled',
+  'ZOOKEEPER_ELECTION_ALG' : 'electionAlg',
+  'ZOOKEEPER_INIT_LIMIT': 'initLimit',
+  'ZOOKEEPER_LEADER_SERVES': 'leaderServes',
+  'ZOOKEEPER_SYNC_LIMIT' : 'syncLimit',
+  'ZOOKEEPER_CNX_TIMEOUT': 'cnxTimeout',
+  'ZOOKEEPER_FORCE_SYNC': 'forceSync',
+  'ZOOKEEPER_JUTE_MAX_BUFFER': 'jute.maxbuffer',
+  'ZOOKEEPER_SKIP_ACL': 'skipACL',
+  'ZOOKEEPER_QUORUM_LISTEN_ON_ALL_IPS': 'quorumListenOnAllIPs'
+ } -%}
+
+{% for k, property in other_props.iteritems() -%}
+{% if env.get(k) != None -%}
+{{property}}={{env[k]}}
+{% endif -%}
+{% endfor -%}
+
+{% if env['ZOOKEEPER_SERVERS'] %}
+{% set servers = env['ZOOKEEPER_SERVERS'].split(';') %}
+{% for server in servers %}
+server.{{ loop.index }}={{server}}
+{% endfor %}
+{% endif %}
+
+{% if env['ZOOKEEPER_GROUPS'] %}
+{% set groups = env['ZOOKEEPER_GROUPS'].split(';') %}
+{% for group in groups %}
+group.{{ loop.index }}={{group}}
+{% endfor %}
+{% endif %}
+
+{% if env['ZOOKEEPER_WEIGHTS'] %}
+{% set weights = env['ZOOKEEPER_WEIGHTS'].split(';') %}
+{% for weight in weights %}
+weight.{{ loop.index }}={{weight}}
+{% endfor %}
+{% endif %}