upgrade cluster to kafka kraft mode

rajadilipkolli · May 12, 2023 · 7fb29df · 7fb29df
1 parent 22244b6
commit 7fb29df
Showing 1 changed file with 24 additions and 36 deletions.
diff --git a/docker-compose-cluster.yml b/docker-compose-cluster.yml
@@ -1,68 +1,59 @@
 version: '3'
 services:
-  zookeeper:
-    image: zookeeper:3.8.1
-    hostname: zookeeper
-    ports:
-      - "2181:2181"
-    environment:
-      ZOO_MY_ID: 1
-      ZOO_PORT: 2181
-      ZOO_SERVERS: server.1=zookeeper:2888:3888
-    volumes:
-      - ./data/zookeeper/data:/data
-      - ./data/zookeeper/datalog:/datalog
 
   kafka1:
-    image: confluentinc/cp-kafka:5.3.8-1
+    image: confluentinc/cp-kafka:7.4.0
     hostname: kafka1
     ports:
       - "9091:9091"
     environment:
+      KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19091,CONTROLLER://kafka1:29093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9091
       KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19091,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9091
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
-      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+      KAFKA_CONTROLLER_QUORUM_MODE: 'kraft'
+      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
       KAFKA_BROKER_ID: 1
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    volumes:
-      - ./data/kafka1/data:/var/lib/kafka/data
-    depends_on:
-      - zookeeper
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka1:29093'
 
   kafka2:
-    image: confluentinc/cp-kafka:5.3.8-1
+    image: confluentinc/cp-kafka:7.4.0
     hostname: kafka2
     ports:
       - "9092:9092"
     environment:
       KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19092,CONTROLLER://kafka2:29093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_CONTROLLER_QUORUM_MODE: 'kraft'
+      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
       KAFKA_BROKER_ID: 2
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    volumes:
-      - ./data/kafka2/data:/var/lib/kafka/data
-    depends_on:
-      - zookeeper 
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '2@kafka2:29093'
 
   kafka3:
-    image: confluentinc/cp-kafka:5.3.8-1
+    image: confluentinc/cp-kafka:7.4.0
     hostname: kafka3
     ports:
       - "9093:9093"
     environment:
       KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
+      KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19093,CONTROLLER://kafka3:29093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
       KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
-      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+      KAFKA_CONTROLLER_QUORUM_MODE: 'kraft'
+      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
       KAFKA_BROKER_ID: 3
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    volumes:
-      - ./data/kafka3/data:/var/lib/kafka/data
-    depends_on:
-      - zookeeper
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '3@kafka3:29093'
 
   kafdrop:
     image: obsidiandynamics/kafdrop
@@ -75,6 +66,3 @@ services:
       - kafka1
       - kafka2
       - kafka3
-
-
-