confluent based image

Issue-ID: DMAAP-1339
Change-Id: Ieb942450ea48ca246876cd0fcf02ba59187cabe2
Signed-off-by: su622b <su622b@att.com>

10 files changed, 289 insertions, 82 deletions

diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile
index 30e3a03..7f2590a 100644
--- a/src/main/docker/Dockerfile
+++ b/src/main/docker/Dockerfile
@@ -1,84 +1,31 @@
-# git helper image
-FROM alpine:3.9 as githelper
-RUN apk add --no-cache git
-# Copy APIKeys
-RUN git init /zookeeper/gerrit \
-    && cd /zookeeper/gerrit \
-    && git remote add origin http://gerrit.onap.org/r/dmaap/messagerouter/messageservice.git \
-    && git config core.sparsecheckout true \
-    && echo "oom-topics/data-zookeeper/*" >> .git/info/sparse-checkout \
-    && git pull --depth=1 origin master
-
-# main image build
-FROM docker.io/zookeeper:${zookeeper.dist.version}
-
-ARG ZK_DIST=zookeeper-${zookeeper.dist.version}
-
-ENV ZK_USER=mrzookeeper \
-    ZK_DATA_DIR=/var/lib/zookeeper/data \
-    ZK_LOG_DIR=/var/log/zookeeper \
-    ZK_CONF_DIR=/opt/zookeeper/conf \
-    ZK_DIST_VER=$ZK_DIST
-
-# passing config path to upstream image
-ENV ZOOCFGDIR=$ZK_CONF_DIR
-
-# copy API-keys from githelper
-COPY --from=githelper /zookeeper/gerrit/oom-topics /tmp/zookeeper/gerrit/oom-topics
-
-# Create required directories and configure file system ownership
-# for necessary directories and symlink the distribution as a user executable
-# and remove unecessary files
-RUN mkdir -p \
-        /opt \
-        /usr/etc \
-        /usr/share/zookeeper \
-        $ZK_DATA_DIR \
-        $ZK_LOG_DIR \
-    && ln -snf /$ZK_DIST_VER/ /opt/zookeeper \
-    && ln -snf /opt/zookeeper/conf/ /usr/etc/zookeeper \
-    && ln -snf /opt/zookeeper/bin/* /usr/bin \
-    && ln -snf /opt/zookeeper/$ZK_DIST_VER.jar /usr/share/zookeeper \
-    && ln -snf /opt/zookeeper/lib/* /usr/share/zookeeper \
-    && rm -rf \
-        /conf \
-        /home/zookeeper \
-        /docker-entrypoint.sh \
-        /opt/zookeeper/CHANGES.txt \
-        /opt/zookeeper/README.txt \
-        /opt/zookeeper/NOTICE.txt \
-        /opt/zookeeper/CHANGES.txt \
-        /opt/zookeeper/README_packaging.txt \
-        /opt/zookeeper/build.xml \
-        /opt/zookeeper/config \
-        /opt/zookeeper/contrib \
-        /opt/zookeeper/dist-maven \
-        /opt/zookeeper/docs \
-        /opt/zookeeper/ivy.xml \
-        /opt/zookeeper/ivysettings.xml \
-        /opt/zookeeper/recipes \
-        /opt/zookeeper/src \
-        /opt/zookeeper/$ZK_DIST_VER.jar.asc \
-        /opt/zookeeper/$ZK_DIST_VER.jar.md5 \
-        /opt/zookeeper/$ZK_DIST_VER.jar.sha1
-
-# Copy configuration generator script to bin
-COPY scripts /opt/zookeeper/bin/
-RUN chmod -R 777 /opt/zookeeper/bin
-
-# delete zookeeper user from upsteram image and create custom user instead
-RUN deluser zookeeper > /dev/null 2>&1 \
-    && { addgroup -S -g 1000 $ZK_USER > /dev/null 2>&1 && adduser -S -D -G $ZK_USER -u 1000 $ZK_USER > /dev/null 2>&1; } \
-    || { groupadd -r -g 1000 $ZK_USER && useradd -r -M -g 1000 -u 1000 $ZK_USER; } \
-    && chown -R $ZK_USER:$ZK_USER \
-        /opt/zookeeper \
-        /tmp/zookeeper \
-        /var/lib/zookeeper \
-        /var/log/zookeeper \
-        $ZK_CONF_DIR \
-        $ZK_DATA_DIR \
-        $ZK_LOG_DIR
+FROM confluentinc/cp-base:5.3.1
+
+ENV COMPONENT=zookeeper \
+    ZK_USER=mrzookeeper
+
+
+
+RUN echo "===> installing ${COMPONENT}..." \
+    && apt-get update && apt-get install -y confluent-kafka-${SCALA_VERSION}=${CONFLUENT_VERSION}${CONFLUENT_PLATFORM_LABEL}-${CONFLUENT_DEB_VERSION} \
+    \
+    && echo "===> clean up ..."  \
+    && apt-get clean && rm -rf /tmp/* /var/lib/apt/lists/* \
+    \
+    && echo "===> Setting up ${COMPONENT} dirs" \
+    && mkdir -p /var/lib/${COMPONENT}/data  /var/lib/${COMPONENT}/log /etc/${COMPONENT}/secrets/jaas /etc/${COMPONENT}/data  /var/log/kafka /var/log/confluent  \
+    && chmod -R ag+w /etc/kafka /var/lib/${COMPONENT}/data /var/lib/${COMPONENT}/log  /etc/${COMPONENT}/secrets /etc/${COMPONENT}/data  /var/log/kafka /var/log/confluent \
+    && chown -R root:root /var/log/kafka /var/log/confluent /var/lib/kafka /var/lib/zookeeper  
+    
+RUN set -x \
+    && apt-get update \
+     && apt-get install -y git \
+     && git clone -b master --single-branch http://gerrit.onap.org/r/dmaap/messagerouter/messageservice.git /tmp/zookeeper/gerrit 	
+COPY include/etc/confluent/docker /etc/confluent/docker
+#COPY zk_server_jaas.conf /etc/${COMPONENT}/secrets/jaas/
+
+RUN useradd  -u 1000  -g 0 $ZK_USER
+USER $ZK_USER
 
 EXPOSE 2181 2888 3888
-USER $ZK_USER
-ENTRYPOINT ["/opt/zookeeper/bin/start-zookeeper.sh"]
\ No newline at end of file
+
+CMD ["/etc/confluent/docker/run"]
diff --git a/src/main/docker/include/etc/confluent/docker/configure b/src/main/docker/include/etc/confluent/docker/configure
new file mode 100644
index 0000000..74e1c11
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/configure
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/confluent/docker/bash-config
+
+dub ensure ZOOKEEPER_CLIENT_PORT
+
+dub path /etc/kafka/ writable
+
+# myid is required for clusters
+if [[ -n "${ZOOKEEPER_SERVERS-}" ]]
+then
+  dub ensure ZOOKEEPER_SERVER_ID
+  export ZOOKEEPER_INIT_LIMIT=${ZOOKEEPER_INIT_LIMIT:-"10"}
+  export ZOOKEEPER_SYNC_LIMIT=${ZOOKEEPER_SYNC_LIMIT:-"5"}
+fi
+
+if [[ -n "${ZOOKEEPER_SERVER_ID-}" ]]
+then
+  dub template "/etc/confluent/docker/myid.template" "/var/lib/${COMPONENT}/data/myid"
+fi
+
+if [[ -n "${KAFKA_JMX_OPTS-}" ]]
+then
+  if [[ ! $KAFKA_JMX_OPTS == *"com.sun.management.jmxremote.rmi.port"*  ]]
+  then
+    echo "KAFKA_JMX_OPTS should contain 'com.sun.management.jmxremote.rmi.port' property. It is required for accessing the JMX metrics externally."
+  fi
+fi
+
+dub template "/etc/confluent/docker/${COMPONENT}.properties.template" "/etc/kafka/${COMPONENT}.properties"
+dub template "/etc/confluent/docker/log4j.properties.template" "/etc/kafka/log4j.properties"
+dub template "/etc/confluent/docker/tools-log4j.properties.template" "/etc/kafka/tools-log4j.properties"
+
+
+ZK_REPLICAS=${ZOOKEEPER_REPLICAS:-1}
+HOST=$(hostname -s)
+DOMAIN=$(hostname -d)
+ZK_SERVER_PORT=${ZOOKEEPER_SERVER_PORT:-2888}
+ZK_ELECTION_PORT=${ZOOKEEPER_ELECTION_PORT:-3888}
+
+function print_servers() {
+   if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
+        NAME=${BASH_REMATCH[1]}
+        for (( i=1; i<=$ZK_REPLICAS; i++ ))
+    do
+        echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT"
+    done
+  
+    fi
+}
+
+
+if [ $ZK_REPLICAS -gt 1 ]; then
+print_servers >> /etc/kafka/${COMPONENT}.properties
+fi
+
+if [ -d /var/lib/zookeeper/data/version-2 ]; then
+          echo "API Keys already loaded";
+          else
+          cp -var /tmp/zookeeper/gerrit/oom-topics/data-zookeeper/* /var/lib/zookeeper/data/;
+          rm -rf /tmp/zookeeper/gerrit;
+          echo "Copying API Keys completed.";
+     fi
+
+
diff --git a/src/main/docker/include/etc/confluent/docker/ensure b/src/main/docker/include/etc/confluent/docker/ensure
new file mode 100644
index 0000000..0f7b8e6
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/ensure
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/confluent/docker/bash-config
+
+echo "===> Check if /var/lib/zookeeper/data is writable ..."
+dub path /var/lib/zookeeper/data writable
+
+echo "===> Check if /var/lib/zookeeper/log is writable ..."
+dub path /var/lib/zookeeper/log writable
diff --git a/src/main/docker/include/etc/confluent/docker/launch b/src/main/docker/include/etc/confluent/docker/launch
new file mode 100644
index 0000000..5cdc97e
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/launch
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Override this section from the script to include the com.sun.management.jmxremote.rmi.port property.
+if [ -z "$KAFKA_JMX_OPTS" ]; then
+  export KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false  -Dcom.sun.management.jmxremote.ssl=false "
+fi
+
+# The JMX client needs to be able to connect to java.rmi.server.hostname.
+# The default for bridged n/w is the bridged IP so you will only be able to connect from another docker container.
+# For host n/w, this is the IP that the hostname on the host resolves to.
+
+# If you have more that one n/w configured, hostname -i gives you all the IPs,
+# the default is to pick the first IP (or network).
+export KAFKA_JMX_HOSTNAME=${KAFKA_JMX_HOSTNAME:-$(hostname -i | cut -d" " -f1)}
+
+if [ "$KAFKA_JMX_PORT" ]; then
+  # This ensures that the "if" section for JMX_PORT in kafka launch script does not trigger.
+  export JMX_PORT=$KAFKA_JMX_PORT
+  export KAFKA_JMX_OPTS="$KAFKA_JMX_OPTS -Djava.rmi.server.hostname=$KAFKA_JMX_HOSTNAME -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.rmi.port=$JMX_PORT -Dcom.sun.management.jmxremote.port=$JMX_PORT"
+fi
+
+
+if [[ -n "${ZOOKEEPER_SERVER_ID-}" ]]
+then
+  echo "===> Printing /var/lib/${COMPONENT}/data/myid "
+  cat /var/lib/"${COMPONENT}"/data/myid
+fi
+
+echo "===> Launching ${COMPONENT} ... "
+exec "${COMPONENT}"-server-start /etc/kafka/"${COMPONENT}".properties
diff --git a/src/main/docker/include/etc/confluent/docker/log4j.properties.template b/src/main/docker/include/etc/confluent/docker/log4j.properties.template
new file mode 100644
index 0000000..972bab3
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/log4j.properties.template
@@ -0,0 +1,13 @@
+
+log4j.rootLogger={{ env["ZOOKEEPER_LOG4J_ROOT_LOGLEVEL"] | default('INFO') }}, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+{% if env['ZOOKEEPER_LOG4J_LOGGERS'] %}
+{% set loggers = parse_log4j_loggers(env['ZOOKEEPER_LOG4J_LOGGERS']) %}
+{% for logger,loglevel in loggers.iteritems() %}
+log4j.logger.{{logger}}={{loglevel}}, stdout
+{% endfor %}
+{% endif %}
diff --git a/src/main/docker/include/etc/confluent/docker/myid.template b/src/main/docker/include/etc/confluent/docker/myid.template
new file mode 100644
index 0000000..d8c94b0
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/myid.template
@@ -0,0 +1 @@
+{{env["ZOOKEEPER_SERVER_ID"]}}
diff --git a/src/main/docker/include/etc/confluent/docker/run b/src/main/docker/include/etc/confluent/docker/run
new file mode 100644
index 0000000..7d311e8
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/run
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Copyright 2016 Confluent Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+. /etc/confluent/docker/bash-config
+
+echo "===> ENV Variables ..."
+show_env
+
+echo "===> User"
+id
+
+echo "===> Configuring ..."
+/etc/confluent/docker/configure
+
+echo "===> Running preflight checks ... "
+/etc/confluent/docker/ensure
+
+echo "===> Launching ... "
+exec /etc/confluent/docker/launch
diff --git a/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template
new file mode 100644
index 0000000..b8ad39c
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/tools-log4j.properties.template
@@ -0,0 +1,7 @@
+
+log4j.rootLogger={{ env["ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL"] | default('WARN') }}, stderr
+
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=[%d] %p %m (%c)%n
+log4j.appender.stderr.Target=System.err
diff --git a/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template b/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template
new file mode 100644
index 0000000..e87e6b6
--- /dev/null
+++ b/src/main/docker/include/etc/confluent/docker/zookeeper.properties.template
@@ -0,0 +1,57 @@
+
+clientPort={{ env['ZOOKEEPER_CLIENT_PORT'] }}
+dataDir=/var/lib/zookeeper/data
+dataLogDir=/var/lib/zookeeper/data
+
+{# optional properties #}
+{% set other_props = {
+  'ZOOKEEPER_TICK_TIME': 'tickTime',
+  'ZOOKEEPER_GLOBAL_OUTSTANDING_LIMIT' : 'globalOutstandingLimit',
+  'ZOOKEEPER_PRE_ALLOC_SIZE': 'preAllocSize',
+  'ZOOKEEPER_SNAP_COUNT': 'snapCount',
+  'ZOOKEEPER_TRACE_FILE': 'traceFile',
+  'ZOOKEEPER_MAX_CLIENT_CNXNS' : 'maxClientCnxns',
+  'ZOOKEEPER_CLIENT_PORT_ADDRESS' : 'clientPortAddress',
+  'ZOOKEEPER_MIN_SESSION_TIMEOUT' : 'minSessionTimeout',
+  'ZOOKEEPER_MAX_SESSION_TIMEOUT' : 'maxSessionTimeout',
+  'ZOOKEEPER_FSYNC_WARNING_THRESHOLDMS' : 'fsync.warningthresholdms',
+  'ZOOKEEPER_AUTOPURGE_SNAP_RETAIN_COUNT' : 'autopurge.snapRetainCount',
+  'ZOOKEEPER_AUTOPURGE_PURGE_INTERVAL': 'autopurge.purgeInterval',
+  'ZOOKEEPER_SYNC_ENABLED': 'syncEnabled',
+  'ZOOKEEPER_ELECTION_ALG' : 'electionAlg',
+  'ZOOKEEPER_INIT_LIMIT': 'initLimit',
+  'ZOOKEEPER_LEADER_SERVES': 'leaderServes',
+  'ZOOKEEPER_SYNC_LIMIT' : 'syncLimit',
+  'ZOOKEEPER_CNX_TIMEOUT': 'cnxTimeout',
+  'ZOOKEEPER_FORCE_SYNC': 'forceSync',
+  'ZOOKEEPER_JUTE_MAX_BUFFER': 'jute.maxbuffer',
+  'ZOOKEEPER_SKIP_ACL': 'skipACL',
+  'ZOOKEEPER_QUORUM_LISTEN_ON_ALL_IPS': 'quorumListenOnAllIPs'
+ } -%}
+
+{% for k, property in other_props.iteritems() -%}
+{% if env.get(k) != None -%}
+{{property}}={{env[k]}}
+{% endif -%}
+{% endfor -%}
+
+{% if env['ZOOKEEPER_SERVERS'] %}
+{% set servers = env['ZOOKEEPER_SERVERS'].split(';') %}
+{% for server in servers %}
+server.{{ loop.index }}={{server}}
+{% endfor %}
+{% endif %}
+
+{% if env['ZOOKEEPER_GROUPS'] %}
+{% set groups = env['ZOOKEEPER_GROUPS'].split(';') %}
+{% for group in groups %}
+group.{{ loop.index }}={{group}}
+{% endfor %}
+{% endif %}
+
+{% if env['ZOOKEEPER_WEIGHTS'] %}
+{% set weights = env['ZOOKEEPER_WEIGHTS'].split(';') %}
+{% for weight in weights %}
+weight.{{ loop.index }}={{weight}}
+{% endfor %}
+{% endif %}
diff --git a/src/main/docker/zk_server_jaas.conf b/src/main/docker/zk_server_jaas.conf
new file mode 100644
index 0000000..0d11df9
--- /dev/null
+++ b/src/main/docker/zk_server_jaas.conf
@@ -0,0 +1,4 @@
+Server {
+       org.apache.zookeeper.server.auth.DigestLoginModule required
+       user_kafka="kafka_secret";
+};