Create Kafka 0.11.0.1 docker image

Issue-ID: DMAAP-628
Change-Id: I3ad0c01d2778870734ccb1ed7daff39afbb18acc
Signed-off-by: sunil unnava <su622b@att.com>

6 files changed, 224 insertions, 0 deletions

diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile
new file mode 100644
index 0000000..39f997c
--- /dev/null
+++ b/src/main/docker/Dockerfile
@@ -0,0 +1,25 @@
+FROM anapsix/alpine-java
+
+ARG kafka_version=0.11.0.1
+ARG scala_version=2.12
+
+
+RUN apk add --update unzip wget curl docker jq coreutils
+
+ENV KAFKA_VERSION=$kafka_version SCALA_VERSION=$scala_version
+ADD download-kafka.sh /tmp/download-kafka.sh
+RUN chmod a+x /tmp/download-kafka.sh && sync && /tmp/download-kafka.sh && tar xfz /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -C /opt && rm /tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz && ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} /opt/kafka
+
+VOLUME ["/kafka"]
+
+ENV KAFKA_HOME /opt/kafka
+ENV PATH ${PATH}:${KAFKA_HOME}/bin
+ADD start-kafka.sh /usr/bin/start-kafka.sh
+ADD broker-list.sh /usr/bin/broker-list.sh
+ADD create-topics.sh /usr/bin/create-topics.sh
+# The scripts need to have executable permission
+RUN chmod a+x /usr/bin/start-kafka.sh && \
+    chmod a+x /usr/bin/broker-list.sh && \
+    chmod a+x /usr/bin/create-topics.sh
+# Use "exec" form so that it runs as PID 1 (useful for graceful shutdown)
+CMD ["start-kafka.sh"]
diff --git a/src/main/docker/broker-list.sh b/src/main/docker/broker-list.sh
new file mode 100644
index 0000000..7f04639
--- /dev/null
+++ b/src/main/docker/broker-list.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}')
+BROKERS=$(for CONTAINER in $CONTAINERS; do docker port $CONTAINER 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done)
+echo $BROKERS | sed -e 's/ /,/g'
diff --git a/src/main/docker/create-topics.sh b/src/main/docker/create-topics.sh
new file mode 100644
index 0000000..34945b3
--- /dev/null
+++ b/src/main/docker/create-topics.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+
+if [[ -z "$START_TIMEOUT" ]]; then
+    START_TIMEOUT=600
+fi
+
+start_timeout_exceeded=false
+count=0
+step=10
+while netstat -lnt | awk '$4 ~ /:'$KAFKA_PORT'$/ {exit 1}'; do
+    echo "waiting for kafka to be ready"
+    sleep $step;
+    count=$(expr $count + $step)
+    if [ $count -gt $START_TIMEOUT ]; then
+        start_timeout_exceeded=true
+        break
+    fi
+done
+
+if $start_timeout_exceeded; then
+    echo "Not able to auto-create topic (waited for $START_TIMEOUT sec)"
+    exit 1
+fi
+
+if [[ -n $KAFKA_CREATE_TOPICS ]]; then
+    IFS=','; for topicToCreate in $KAFKA_CREATE_TOPICS; do
+        echo "creating topics: $topicToCreate"
+        IFS=':' read -a topicConfig <<< "$topicToCreate"
+        if [ ${topicConfig[3]} ]; then
+          JMX_PORT='' $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $KAFKA_ZOOKEEPER_CONNECT --replication-factor ${topicConfig[2]} --partitions ${topicConfig[1]} --topic "${topicConfig[0]}" --config cleanup.policy="${topicConfig[3]}" --if-not-exists
+        else
+          JMX_PORT='' $KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper $KAFKA_ZOOKEEPER_CONNECT --replication-factor ${topicConfig[2]} --partitions ${topicConfig[1]} --topic "${topicConfig[0]}" --if-not-exists
+        fi
+    done
+fi
diff --git a/src/main/docker/docker-compose.yml b/src/main/docker/docker-compose.yml
new file mode 100644
index 0000000..04b82c3
--- /dev/null
+++ b/src/main/docker/docker-compose.yml
@@ -0,0 +1,15 @@
+version: '2'
+services:
+  zookeeper:
+    image: wurstmeister/zookeeper
+    ports:
+      - "2181:2181"
+  kafka:
+    build: .
+    ports:
+      - "9092"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
diff --git a/src/main/docker/download-kafka.sh b/src/main/docker/download-kafka.sh
new file mode 100644
index 0000000..2ddc911
--- /dev/null
+++ b/src/main/docker/download-kafka.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+mirror=$(curl --stderr /dev/null https://www.apache.org/dyn/closer.cgi\?as_json\=1 | jq -r '.preferred')
+url="${mirror}kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"
+wget -q "${url}" -O "/tmp/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz"
diff --git a/src/main/docker/start-kafka.sh b/src/main/docker/start-kafka.sh
new file mode 100644
index 0000000..5571a59
--- /dev/null
+++ b/src/main/docker/start-kafka.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+if [[ -z "$KAFKA_PORT" ]]; then
+    export KAFKA_PORT=9092
+fi
+
+create-topics.sh &
+
+if [[ -z "$KAFKA_ADVERTISED_PORT" && \
+  -z "$KAFKA_LISTENERS" && \
+  -z "$KAFKA_ADVERTISED_LISTENERS" && \
+  -S /var/run/docker.sock ]]; then
+    export KAFKA_ADVERTISED_PORT=$(docker port `hostname` $KAFKA_PORT | sed -r "s/.*:(.*)/\1/g")
+fi
+if [[ -z "$KAFKA_BROKER_ID" ]]; then
+    if [[ -n "$BROKER_ID_COMMAND" ]]; then
+        export KAFKA_BROKER_ID=$(eval $BROKER_ID_COMMAND)
+    else
+        # By default auto allocate broker ID
+        export KAFKA_BROKER_ID=-1
+    fi
+fi
+if [[ -z "$KAFKA_LOG_DIRS" ]]; then
+    export KAFKA_LOG_DIRS="/kafka/kafka-logs-$HOSTNAME"
+fi
+if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then
+    export KAFKA_ZOOKEEPER_CONNECT=$(env | grep ZK.*PORT_2181_TCP= | sed -e 's|.*tcp://||' | paste -sd ,)
+fi
+
+if [[ -n "$KAFKA_HEAP_OPTS" ]]; then
+    sed -r -i "s/(export KAFKA_HEAP_OPTS)=\"(.*)\"/\1=\"$KAFKA_HEAP_OPTS\"/g" $KAFKA_HOME/bin/kafka-server-start.sh
+    unset KAFKA_HEAP_OPTS
+fi
+
+if [[ -z "$KAFKA_ADVERTISED_HOST_NAME" && -n "$HOSTNAME_COMMAND" ]]; then
+    export KAFKA_ADVERTISED_HOST_NAME=$(eval $HOSTNAME_COMMAND)
+fi
+
+if [[ -n "$KAFKA_LISTENER_SECURITY_PROTOCOL_MAP" ]]; then
+  if [[ -n "$KAFKA_ADVERTISED_PORT" && -n "$KAFKA_ADVERTISED_PROTOCOL_NAME" ]]; then
+    export KAFKA_ADVERTISED_LISTENERS="${KAFKA_ADVERTISED_PROTOCOL_NAME}://${KAFKA_ADVERTISED_HOST_NAME-}:${KAFKA_ADVERTISED_PORT}"
+    export KAFKA_LISTENERS="$KAFKA_ADVERTISED_PROTOCOL_NAME://:$KAFKA_ADVERTISED_PORT"
+  fi
+
+  if [[ -z "$KAFKA_PROTOCOL_NAME" ]]; then
+    export KAFKA_PROTOCOL_NAME="${KAFKA_ADVERTISED_PROTOCOL_NAME}"
+  fi
+
+  if [[ -n "$KAFKA_PORT" && -n "$KAFKA_PROTOCOL_NAME" ]]; then
+    export ADD_LISTENER="${KAFKA_PROTOCOL_NAME}://${KAFKA_HOST_NAME-}:${KAFKA_PORT}"
+  fi
+
+  if [[ -z "$KAFKA_INTER_BROKER_LISTENER_NAME" ]]; then
+    export KAFKA_INTER_BROKER_LISTENER_NAME=$KAFKA_PROTOCOL_NAME
+  fi
+else
+   #DEFAULT LISTENERS 
+   export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://${KAFKA_ADVERTISED_HOST_NAME-}:${KAFKA_ADVERTISED_PORT-$KAFKA_PORT}"
+   export KAFKA_LISTENERS="PLAINTEXT://${KAFKA_HOST_NAME-}:${KAFKA_PORT-9092}"
+fi
+
+if [[ -n "$ADD_LISTENER" && -n "$KAFKA_LISTENERS" ]]; then
+  export KAFKA_LISTENERS="${KAFKA_LISTENERS},${ADD_LISTENER}"
+fi
+
+if [[ -n "$ADD_LISTENER" && -z "$KAFKA_LISTENERS" ]]; then
+  export KAFKA_LISTENERS="${ADD_LISTENER}"
+fi
+
+if [[ -n "$ADD_LISTENER" && -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then
+  export KAFKA_ADVERTISED_LISTENERS="${KAFKA_ADVERTISED_LISTENERS},${ADD_LISTENER}"
+fi
+
+if [[ -n "$ADD_LISTENER" && -z "$KAFKA_ADVERTISED_LISTENERS" ]]; then
+  export KAFKA_ADVERTISED_LISTENERS="${ADD_LISTENER}"
+fi
+
+if [[ -n "$KAFKA_INTER_BROKER_LISTENER_NAME" && ! "$KAFKA_INTER_BROKER_LISTENER_NAME"X = "$KAFKA_PROTOCOL_NAME"X ]]; then
+   if [[ -n "$KAFKA_INTER_BROKER_PORT" ]]; then
+      export KAFKA_INTER_BROKER_PORT=$(( $KAFKA_PORT + 1 ))
+   fi
+   export INTER_BROKER_LISTENER="${KAFKA_INTER_BROKER_LISTENER_NAME}://:${KAFKA_INTER_BROKER_PORT}"
+   export KAFKA_LISTENERS="${KAFKA_LISTENERS},${INTER_BROKER_LISTENER}"
+   export KAFKA_ADVERTISED_LISTENERS="${KAFKA_ADVERTISED_LISTENERS},${INTER_BROKER_LISTENER}"
+   unset KAFKA_INTER_BROKER_PORT
+   unset KAFKA_SECURITY_INTER_BROKER_PROTOCOL
+   unset INTER_BROKER_LISTENER
+fi
+
+if [[ -n "$RACK_COMMAND" && -z "$KAFKA_BROKER_RACK" ]]; then
+    export KAFKA_BROKER_RACK=$(eval $RACK_COMMAND)
+fi
+
+#Issue newline to config file in case there is not one already
+echo -e "\n" >> $KAFKA_HOME/config/server.properties
+
+unset KAFKA_CREATE_TOPICS
+unset KAFKA_ADVERTISED_PROTOCOL_NAME
+unset KAFKA_PROTOCOL_NAME
+
+if [[ -n "$KAFKA_ADVERTISED_LISTENERS" ]]; then
+  unset KAFKA_ADVERTISED_PORT
+  unset KAFKA_ADVERTISED_HOST_NAME
+fi
+
+if [[ -n "$KAFKA_LISTENERS" ]]; then
+  unset KAFKA_PORT
+  unset KAFKA_HOST_NAME
+fi
+
+for VAR in `env`
+do
+  if [[ $VAR =~ ^KAFKA_ && ! $VAR =~ ^KAFKA_HOME ]]; then
+    kafka_name=`echo "$VAR" | sed -r "s/KAFKA_(.*)=.*/\1/g" | tr '[:upper:]' '[:lower:]' | tr _ .`
+    env_var=`echo "$VAR" | sed -r "s/(.*)=.*/\1/g"`
+    if egrep -q "(^|^#)$kafka_name=" $KAFKA_HOME/config/server.properties; then
+        sed -r -i "s@(^|^#)($kafka_name)=(.*)@\2=${!env_var}@g" $KAFKA_HOME/config/server.properties #note that no config values may contain an '@' char
+    else
+        echo "$kafka_name=${!env_var}" >> $KAFKA_HOME/config/server.properties
+    fi
+  fi
+
+  if [[ $VAR =~ ^LOG4J_ ]]; then
+    log4j_name=`echo "$VAR" | sed -r "s/(LOG4J_.*)=.*/\1/g" | tr '[:upper:]' '[:lower:]' | tr _ .`
+    log4j_env=`echo "$VAR" | sed -r "s/(.*)=.*/\1/g"`
+    if egrep -q "(^|^#)$log4j_name=" $KAFKA_HOME/config/log4j.properties; then
+        sed -r -i "s@(^|^#)($log4j_name)=(.*)@\2=${!log4j_env}@g" $KAFKA_HOME/config/log4j.properties #note that no config values may contain an '@' char
+    else
+        echo "$log4j_name=${!log4j_env}" >> $KAFKA_HOME/config/log4j.properties
+    fi
+  fi
+done
+
+if [[ -n "$CUSTOM_INIT_SCRIPT" ]] ; then
+  eval $CUSTOM_INIT_SCRIPT
+fi
+
+exec $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties