kafka教程（新手使用单机版）

文章目录

一、概念
二、安装
三、使用

一、概念

kafka是一个分布式消息系统、所谓消息系统主要功能就是发布/订阅。

二、安装

为了省事，直接docker安装了，docker-compose.yml文件如下

概念说明：环境中一共包含四个

1、zookeeper：kafka数据调度，关键组件

2、kafka组件

3、kafka-connect

4、kafka-ui：kafka图形可视化

版本说明：kafka采用稳定版7.4.1，最新版有KRaft能够替代zookeeper，为了保险起见还是先用稳定版，如果用最新版就不用安装zookeeper

yaml 复制代码

services:
  # Zookeeper 服务
  zookeeper:
    image: docker.1ms.run/library/zookeeper:latest
    container_name: zookeeper
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
    ports:
      - "2181:2181"
    restart: unless-stopped
    volumes:
      - kafka_data:/bitnami/zookeeper

  # Kafka 服务
  kafka:
    image: docker.1ms.run/confluentinc/cp-kafka:7.4.1
    container_name: kafka
    environment:
      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"

      # 监听器配置 (核心)
      KAFKA_LISTENERS: "PLAINTEXT://:9092"
      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092"

      # 单节点环境必需的关键配置
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
      KAFKA_NUM_PARTITIONS: 3

      # 其他推荐配置
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_LOG_RETENTION_HOURS: 168
      KAFKA_LOG_SEGMENT_BYTES: 1073741824
      # Confluent 镜像特定优化
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: localhost
    ports:
      - "9092:9092"
      - "29092:29092"
    depends_on:
      - zookeeper
    restart: unless-stopped
    volumes:
      - kafka_data:/bitnami/kafka

  # Kafka Connect 服务
  kafka-connect:
    image: docker.1ms.run/confluentinc/cp-kafka-connect:latest
    container_name: kafka-connect
    environment:
      - CONNECT_BOOTSTRAP_SERVERS=kafka:9092
      - CONNECT_GROUP_ID=kafka-connect-cluster
      - CONNECT_CONFIG_STORAGE_TOPIC=_connect-configs
      - CONNECT_OFFSET_STORAGE_TOPIC=_connect-offsets
      - CONNECT_STATUS_STORAGE_TOPIC=_connect-status
      - CONNECT_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE=false
      - CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE=false
      - CONNECT_REST_ADVERTISED_HOST_NAME=kafka-connect
      - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR=1
      - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR=1
      - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR=1
      - CONNECT_PLUGIN_PATH=/usr/share/java,/usr/share/confluent-hub-components
    ports:
      - "8083:8083"
    depends_on:
      - kafka
    restart: unless-stopped
    volumes:
      - kafka_data:/usr/share/confluent-hub-components
  kafka-ui:
    image: docker.1ms.run/provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    ports:
      - "8080:8080"  # 通过浏览器访问 http://localhost:8080
    environment:
      # 动态配置模式，允许在Web界面中添加集群
      - DYNAMIC_CONFIG_ENABLED=true
      # 也可以通过环境变量直接配置集群连接（示例）
      #- KAFKA_CLUSTERS_0_NAME=local
      #- KAFKA_CLUSTERS_0_BOOTSTRAP_SERVERS=kafka:9092
      #- KAFKA_CLUSTERS_0_ZOOKEEPER=zookeeper:2181
      #- KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME=local-connect
      #- KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS=http://kafka-connect:8083
    depends_on:
      - kafka
      - kafka-connect
volumes:
  kafka_data:

三、使用

使用kafka-ui地址：http://localhost:8080/

添加管理kafka集群和kafka-connect

添加成功：kafka-connect此时并没有显示是因为没有配置连接插件