【软件部署】用docker部署Apache Kafka 集群架构isolated模式带SSL

说明

带SSL，增强安全

配置

创建文件compose.yaml

bash 复制代码

# 有SSL

services:
  init-kafka-perms:
    image: busybox:latest
    command: sh -c "chown -R 1000:1000 /controller-1 /controller-2 /controller-3 /kafka1 /kafka2 /kafka3"
    volumes:
      - controller-1:/controller-1
      - controller-2:/controller-2
      - controller-3:/controller-3
      - kafka1-logs:/kafka1
      - kafka2-logs:/kafka2
      - kafka3-logs:/kafka3
    restart: "no"
    container_name: kafka-perms-fix
    networks:
      - kafka

  controller-1:
    image: apache/kafka:4.2.0
    hostname: controller-1
    container_name: kafka-controller-1
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: 'controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LISTENERS: 'CONTROLLER://:29092'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      init-kafka-perms:
        condition: service_completed_successfully
    networks:
      - kafka
    volumes:
      - controller-1:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 29092 || exit 1
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s
  
  controller-2:
    image: apache/kafka:4.2.0
    hostname: controller-2
    container_name: kafka-controller-2
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 2
      KAFKA_PROCESS_ROLES: 'controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LISTENERS: 'CONTROLLER://:29092'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      init-kafka-perms:
        condition: service_completed_successfully
    networks:
      - kafka
    volumes:
      - controller-2:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 29092 || exit 1
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s
    
  controller-3:
    image: apache/kafka:4.2.0
    hostname: controller-3
    container_name: kafka-controller-3
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 3
      KAFKA_PROCESS_ROLES: 'controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_LISTENERS: 'CONTROLLER://:29092'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
    depends_on:
      init-kafka-perms:
        condition: service_completed_successfully
    networks:
      - kafka
    volumes:
      - controller-3:/tmp/kraft-combined-logs
    healthcheck:
      test: nc -z localhost 29092 || exit 1
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 10s

  kafka-1:
    image: apache/kafka:4.2.0
    hostname: kafka-1
    container_name: kafka-1
    ports:
      - 29093:9093
    volumes:
      - ./secrets:/etc/kafka/secrets
      - kafka1-logs:/tmp/kraft-combined-logs
    environment:
      KAFKA_NODE_ID: 4
      KAFKA_PROCESS_ROLES: 'broker'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SSL:SSL,CONTROLLER:PLAINTEXT,SSL-INTERNAL:SSL'
      KAFKA_LISTENERS: 'SSL-INTERNAL://:19093,SSL://:9093'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL-INTERNAL'
      KAFKA_SECURITY_PROTOCOL: SSL
      KAFKA_ADVERTISED_LISTENERS: SSL-INTERNAL://kafka-1:19093,SSL://localhost:29093
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
      KAFKA_SSL_KEYSTORE_FILENAME: 'kafka01.keystore.jks'
      KAFKA_SSL_KEYSTORE_CREDENTIALS: 'kafka_keystore_creds'
      KAFKA_SSL_KEY_CREDENTIALS: 'kafka_ssl_key_creds'
      KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.truststore.jks'
      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'kafka_truststore_creds'
      KAFKA_SSL_CLIENT_AUTH: 'required'
      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
    depends_on:
      controller-1: { condition: service_healthy }
      controller-2: { condition: service_healthy }
      controller-3: { condition: service_healthy }
    restart: unless-stopped
    networks:
      - kafka
    healthcheck:
      test: nc -z localhost 9093 || exit 1
      interval: 60s
      timeout: 5s
      retries: 2
      start_period: 30s

  kafka-2:
    image: apache/kafka:4.2.0
    hostname: kafka-2
    container_name: kafka-2
    ports:
      - 39093:9093
    volumes:
      - ./secrets:/etc/kafka/secrets
      - kafka2-logs:/tmp/kraft-combined-logs
    environment:
      KAFKA_NODE_ID: 5
      KAFKA_PROCESS_ROLES: 'broker'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SSL:SSL,CONTROLLER:PLAINTEXT,SSL-INTERNAL:SSL'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
      KAFKA_LISTENERS: 'SSL-INTERNAL://:19093,SSL://:9093'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL-INTERNAL'
      KAFKA_SECURITY_PROTOCOL: SSL
      KAFKA_ADVERTISED_LISTENERS: SSL-INTERNAL://kafka-2:19093,SSL://localhost:39093
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
      KAFKA_SSL_KEYSTORE_FILENAME: 'kafka01.keystore.jks'
      KAFKA_SSL_KEYSTORE_CREDENTIALS: 'kafka_keystore_creds'
      KAFKA_SSL_KEY_CREDENTIALS: 'kafka_ssl_key_creds'
      KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.truststore.jks'
      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'kafka_truststore_creds'
      KAFKA_SSL_CLIENT_AUTH: 'required'
      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
    depends_on:
      controller-1: { condition: service_healthy }
      controller-2: { condition: service_healthy }
      controller-3: { condition: service_healthy }
    restart: unless-stopped
    networks:
      - kafka
    healthcheck:
      test: nc -z localhost 9093 || exit 1
      interval: 60s
      timeout: 5s
      retries: 2
      start_period: 30s

  kafka-3:
    image: apache/kafka:4.2.0
    hostname: kafka-3
    container_name: kafka-3
    ports:
      - 49093:9093
    volumes:
      - ./secrets:/etc/kafka/secrets
      - kafka3-logs:/tmp/kraft-combined-logs
    environment:
      KAFKA_NODE_ID: 6
      KAFKA_PROCESS_ROLES: 'broker'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'SSL:SSL,CONTROLLER:PLAINTEXT,SSL-INTERNAL:SSL'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@controller-1:29092,2@controller-2:29092,3@controller-3:29092'
      KAFKA_LISTENERS: 'SSL-INTERNAL://:19093,SSL://:9093'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'SSL-INTERNAL'
      KAFKA_SECURITY_PROTOCOL: SSL
      KAFKA_ADVERTISED_LISTENERS: SSL-INTERNAL://kafka-3:19093,SSL://localhost:49093
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      CLUSTER_ID: '4L6g3nShT-eMCtK--X86sw'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_REPLICATION_FACTOR: 3
      KAFKA_SHARE_COORDINATOR_STATE_TOPIC_MIN_ISR: 2
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
      KAFKA_SSL_KEYSTORE_FILENAME: 'kafka01.keystore.jks'
      KAFKA_SSL_KEYSTORE_CREDENTIALS: 'kafka_keystore_creds'
      KAFKA_SSL_KEY_CREDENTIALS: 'kafka_ssl_key_creds'
      KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.truststore.jks'
      KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'kafka_truststore_creds'
      KAFKA_SSL_CLIENT_AUTH: 'required'
      KAFKA_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: ""
    depends_on:
      controller-1: { condition: service_healthy }
      controller-2: { condition: service_healthy }
      controller-3: { condition: service_healthy }
    restart: unless-stopped
    networks:
      - kafka
    healthcheck:
      test: nc -z localhost 9093 || exit 1
      interval: 60s
      timeout: 5s
      retries: 2
      start_period: 30s

volumes:
  controller-1:
    name: kafka-controller-1
  controller-2:
    name: kafka-controller-2
  controller-3:
    name: kafka-controller-3
  kafka1-logs:
    name: kafka1-logs
  kafka2-logs:
    name: kafka2-logs
  kafka3-logs:
    name: kafka3-logs

networks:
  kafka:
    name: kafka

创建目录secrets

自己生成文件或下载SSL模板文件https://github.com/apache/kafka/tree/trunk/docker/examples/fixtures/secrets

部署

bash 复制代码

docker compose up -d

测试

确认容器健康

功能测试待更新。