From 3e87d1451f8e233b9396662355b740343cd505e2 Mon Sep 17 00:00:00 2001 From: atompi Date: Wed, 21 Jul 2021 10:37:55 +0800 Subject: [PATCH 1/3] support kafka kraft without zookeeper --- Dockerfile | 5 +++-- start-kafka.sh | 19 +++++++++++++++---- test/docker-compose.yml | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 82b4ee6d..372c5a28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM azul/zulu-openjdk-alpine:8u292-8.54.0.21 -ARG kafka_version=2.7.0 +ARG kafka_version=2.8.0 ARG scala_version=2.13 ARG glibc_version=2.31-r0 ARG vcs_ref=unspecified @@ -24,7 +24,8 @@ ENV PATH=${PATH}:${KAFKA_HOME}/bin COPY download-kafka.sh start-kafka.sh broker-list.sh create-topics.sh versions.sh /tmp/ -RUN apk add --no-cache bash curl jq docker \ +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories \ + && apk add --no-cache bash curl jq docker \ && chmod a+x /tmp/*.sh \ && mv /tmp/start-kafka.sh /tmp/broker-list.sh /tmp/create-topics.sh /tmp/versions.sh /usr/bin \ && sync && /tmp/download-kafka.sh \ diff --git a/start-kafka.sh b/start-kafka.sh index 56927db4..5eda948b 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -10,7 +10,7 @@ fi # Store original IFS config, so we can restore it at various stages ORIG_IFS=$IFS -if [[ -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then +if [[ -z "$KAFKA_WITHOUT_ZOOKEEPER" && -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then echo "ERROR: missing mandatory config: KAFKA_ZOOKEEPER_CONNECT" exit 1 fi @@ -118,7 +118,7 @@ echo "" >> "$KAFKA_HOME/config/server.properties" # Fixes #312 # KAFKA_VERSION + KAFKA_HOME + grep -rohe KAFKA[A-Z0-0_]* /opt/kafka/bin | sort | uniq | tr '\n' '|' - EXCLUSIONS="|KAFKA_VERSION|KAFKA_HOME|KAFKA_DEBUG|KAFKA_GC_LOG_OPTS|KAFKA_HEAP_OPTS|KAFKA_JMX_OPTS|KAFKA_JVM_PERFORMANCE_OPTS|KAFKA_LOG|KAFKA_OPTS|" + EXCLUSIONS="|KAFKA_VERSION|KAFKA_HOME|KAFKA_DEBUG|KAFKA_GC_LOG_OPTS|KAFKA_HEAP_OPTS|KAFKA_JMX_OPTS|KAFKA_JVM_PERFORMANCE_OPTS|KAFKA_LOG|KAFKA_OPTS|KAFKA_WITHOUT_ZOOKEEPER|" # Read in env as a new-line separated array. This handles the case of env variables have spaces and/or carriage returns. See #313 IFS=$'\n' @@ -132,7 +132,11 @@ echo "" >> "$KAFKA_HOME/config/server.properties" if [[ $env_var =~ ^KAFKA_ ]]; then kafka_name=$(echo "$env_var" | cut -d_ -f2- | tr '[:upper:]' '[:lower:]' | tr _ .) - updateConfig "$kafka_name" "${!env_var}" "$KAFKA_HOME/config/server.properties" + if [[ -z "$KAFKA_WITHOUT_ZOOKEEPER" ]]; then + updateConfig "$kafka_name" "${!env_var}" "$KAFKA_HOME/config/server.properties" + else + updateConfig "$kafka_name" "${!env_var}" "$KAFKA_HOME/config/kraft/server.properties" + fi fi if [[ $env_var =~ ^LOG4J_ ]]; then @@ -146,4 +150,11 @@ if [[ -n "$CUSTOM_INIT_SCRIPT" ]] ; then eval "$CUSTOM_INIT_SCRIPT" fi -exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/server.properties" +if [[ -z "$KAFKA_WITHOUT_ZOOKEEPER" ]]; then + exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/server.properties" +elif [[ -d "/kafka/kafka-logs-kafka1" ]]; then + exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/kraft/server.properties" +else + bash -c "$KAFKA_HOME/bin/kafka-storage.sh format -t $($KAFKA_HOME/bin/kafka-storage.sh random-uuid) -c $KAFKA_HOME/config/kraft/server.properties" + exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/kraft/server.properties" +fi diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 42a7ca59..69187b9b 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -49,7 +49,7 @@ services: image: confluentinc/cp-kafkacat:5.0.0 environment: - BROKER_LIST - - KAFKA_VERSION=${KAFKA_VERSION-2.7.0} + - KAFKA_VERSION=${KAFKA_VERSION-2.8.0} volumes: - .:/tests working_dir: /tests From bd736e0133170d02d6599f035f6aba3b1b0c2849 Mon Sep 17 00:00:00 2001 From: atompi Date: Wed, 21 Jul 2021 11:41:44 +0800 Subject: [PATCH 2/3] Update README --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index d55939ad..90e63768 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,16 @@ Older compose files using the short-version of port mapping may encounter Kafka See the included sample compose file ```docker-compose-swarm.yml``` +## Without Zookeeper + +If you need to enable Kraft support, set the following environments (3 nodes): + +``` +KAFKA_WITHOUT_ZOOKEEPER: "true" +KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://:9093 +KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka1:9093,2@kafka2:9093,3@kafka3:9093 +``` + ## Release process See the [wiki](https://github.com/wurstmeister/kafka-docker/wiki/ReleaseProcess) for information on adding or updating versions to release to Dockerhub. From 58b593d618be8b4e4ab9c04b9c50a9b3cb8077bc Mon Sep 17 00:00:00 2001 From: atompi Date: Fri, 23 Jul 2021 17:57:10 +0800 Subject: [PATCH 3/3] support kafka kraft without zookeeper --- README.md | 11 ++++++++++- start-kafka.sh | 7 ++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 90e63768..b9e33d1c 100644 --- a/README.md +++ b/README.md @@ -217,10 +217,19 @@ See the included sample compose file ```docker-compose-swarm.yml``` If you need to enable Kraft support, set the following environments (3 nodes): +first, Generate a cluster ID + +``` +docker run --rm wurstmeister/kafka:2.8.0 kafka-storage.sh random-uuid +``` + +then, set the following environments (3 nodes): + ``` KAFKA_WITHOUT_ZOOKEEPER: "true" KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://:9093 -KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka1:9093,2@kafka2:9093,3@kafka3:9093 +KAFKA_CONTROLLER_QUORUM_VOTERS: 1\@kafka1:9093,2\@kafka2:9093,3\@kafka3:9093 +CLUSTER_UUID: ``` ## Release process diff --git a/start-kafka.sh b/start-kafka.sh index 5eda948b..4a6ac2c8 100755 --- a/start-kafka.sh +++ b/start-kafka.sh @@ -15,6 +15,11 @@ if [[ -z "$KAFKA_WITHOUT_ZOOKEEPER" && -z "$KAFKA_ZOOKEEPER_CONNECT" ]]; then exit 1 fi +if [[ ! -z "$KAFKA_WITHOUT_ZOOKEEPER" && -z "$CLUSTER_UUID" ]]; then + echo "ERROR: missing CLUSTER_UUID when use KAFKA_WITHOUT_ZOOKEEPER" + exit 1 +fi + if [[ -z "$KAFKA_PORT" ]]; then export KAFKA_PORT=9092 fi @@ -155,6 +160,6 @@ if [[ -z "$KAFKA_WITHOUT_ZOOKEEPER" ]]; then elif [[ -d "/kafka/kafka-logs-kafka1" ]]; then exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/kraft/server.properties" else - bash -c "$KAFKA_HOME/bin/kafka-storage.sh format -t $($KAFKA_HOME/bin/kafka-storage.sh random-uuid) -c $KAFKA_HOME/config/kraft/server.properties" + bash -c "$KAFKA_HOME/bin/kafka-storage.sh format -t $CLUSTER_UUID -c $KAFKA_HOME/config/kraft/server.properties" exec "$KAFKA_HOME/bin/kafka-server-start.sh" "$KAFKA_HOME/config/kraft/server.properties" fi