kafka的替代品redpanda部署与SpringBoot集成使用案例

一、Kafka与Redpanda对比分析

1.1 核心差异

特性	Apache Kafka	Redpanda
架构	JVM-based，需要ZooKeeper	C++编写，无外部依赖
性能	高吞吐量，相对较高延迟	更高吞吐量，更低延迟
资源占用	较高（JVM开销）	更低（原生编译）
部署复杂度	需要ZooKeeper协调	单二进制文件，简化部署
兼容性	原生Kafka协议	完全兼容Kafka协议
运维	成熟工具链	简化运维，内置监控

1.2 适用场景

选择Kafka的情况：

• 已有Kafka技术栈和运维经验
• 需要成熟的生态系统和工具链
• 大规模企业级部署

选择Redpanda的情况：

• 追求更高性能和更低延迟
• 希望简化部署和运维
• 资源受限环境
• 快速原型开发

二、Redpanda部署配置

version: '3.3'

services:
  redpanda-0:
    command:
      - redpanda
      - start
      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
      - --advertise-kafka-addr internal://redpanda-0:9092,external://192.168.8.108:19092
      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
      - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
      - --rpc-addr redpanda-0:33145
      - --advertise-rpc-addr redpanda-0:33145
      - --mode dev-container
      - --smp 1
      - --default-log-level=info
    image: redpandadata/redpanda:v25.3.1
    container_name: redpanda-0
    volumes:
      - redpanda-0:/var/lib/redpanda/data
    networks:
      - redpanda_network
    ports:
      - 18081:18081
      - 18082:18082
      - 19092:19092
      - 19644:9644

  console:
    container_name: redpanda-console
    image: redpandadata/console:v3.3.1
    networks:
      - redpanda_network
    entrypoint: /bin/sh
    command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
    environment:
      CONFIG_FILEPATH: /tmp/config.yml
      CONSOLE_CONFIG_FILE: |
        kafka:
          brokers: ["redpanda-0:9092"]
        schemaRegistry:
          enabled: true
          urls: ["http://redpanda-0:8081"]
        redpanda:
          adminApi:
            enabled: true
            urls: ["http://redpanda-0:9644"]
    ports:
      - 8080:8080
    depends_on:
      - redpanda-0

networks:
  redpanda_network:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.101.0.0/16

volumes:
  redpanda-0:

2.2 启动命令

# 启动服务
docker-compose up -d

# 查看服务状态
docker-compose ps

# 查看日志
docker-compose logs -f redpanda

# 停止服务
docker-compose down

三、Spring Boot 3集成案例

3.1 项目结构

src/
├── main/
│   ├── java/
│   │   └── com/example/redpandademo/
│   │       ├── RedpandaDemoApplication.java
│   │       ├── config/
│   │       │   └── KafkaConfig.java
│   │       ├── controller/
│   │       │   └── DemoController.java
│   │       ├── service/
│   │       │   ├── KafkaProducerService.java
│   │       │   └── KafkaConsumerService.java
│   │       └── events/
│   │           └── UserEvent.java
│   └── resources/
│       └── application.yml
└── test/
    └── java/
        └── com/example/redpandademo/
            └── RedpandaDemoApplicationTests.java

3.2 Maven依赖配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0">
    <modelVersion>4.0.0</modelVersion>
    
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>3.2.0</version>
        <relativePath/>
    </parent>
    
    <groupId>com.example</groupId>
    <artifactId>redpanda-demo</artifactId>
    <version>1.0.0</version>
    
    <properties>
        <java.version>17</java.version>
        <confluent.version>7.5.1</confluent.version>
    </properties>
    
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>
        
        <dependency>
            <groupId>io.confluent</groupId>
            <artifactId>kafka-avro-serializer</artifactId>
            <version>${confluent.version}</version>
        </dependency>
        
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        
        <dependency>
            <groupId>org.testcontainers</groupId>
            <artifactId>kafka</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>
    
    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

3.3 应用配置

application.yml

spring:
  application:
    name: redpanda-demo
  kafka:
    bootstrap-servers: localhost:19092
    properties:
      schema.registry.url: http://localhost:18081
    
    producer:
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      value-serializer: org.springframework.kafka.support.serializer.JsonSerializer
      acks: all
      properties:
        retries: 3
        linger.ms: 10
        batch.size: 16384
    
    consumer:
      group-id: redpanda-demo-group
      auto-offset-reset: earliest
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.springframework.kafka.support.serializer.JsonDeserializer
      properties:
        spring.json.trusted.packages: "com.example.redpandademo.events"

server:
  port: 8080

logging:
  level:
    com.example.redpandademo: DEBUG
    org.apache.kafka: WARN

3.4 配置类

KafkaConfig.java

package com.example.redpandademo.config;

import org.apache.kafka.clients.admin.NewTopic;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.config.TopicBuilder;

@Configuration
public class KafkaConfig {
    
    @Bean
    public NewTopic userEventsTopic() {
        return TopicBuilder.name("user-events")
                .partitions(3)
                .replicas(1)
                .build();
    }
    
    @Bean
    public NewTopic orderEventsTopic() {
        return TopicBuilder.name("order-events")
                .partitions(5)
                .replicas(1)
                .build();
    }
}

3.5 消息实体类

UserEvent.java

package com.example.redpandademo.events;

import java.time.LocalDateTime;

public class UserEvent {
    private String userId;
    private String eventType;
    private String email;
    private LocalDateTime timestamp;
    
    public UserEvent() {}
    
    public UserEvent(String userId, String eventType, String email) {
        this.userId = userId;
        this.eventType = eventType;
        this.email = email;
        this.timestamp = LocalDateTime.now();
    }
    
    // Getter和Setter方法
    public String getUserId() { return userId; }
    public void setUserId(String userId) { this.userId = userId; }
    
    public String getEventType() { return eventType; }
    public void setEventType(String eventType) { this.eventType = eventType; }
    
    public String getEmail() { return email; }
    public void setEmail(String email) { this.email = email; }
    
    public LocalDateTime getTimestamp() { return timestamp; }
    public void setTimestamp(LocalDateTime timestamp) { this.timestamp = timestamp; }
    
    @Override
    public String toString() {
        return String.format("UserEvent{userId='%s', eventType='%s', email='%s', timestamp=%s}",
                userId, eventType, email, timestamp);
    }
}

3.6 消息生产者服务

KafkaProducerService.java

package com.example.redpandademo.service;

import com.example.redpandademo.events.UserEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Service;

import java.util.concurrent.CompletableFuture;

@Service
public class KafkaProducerService {
    
    private static final Logger log = LoggerFactory.getLogger(KafkaProducerService.class);
    
    private final KafkaTemplate<String, Object> kafkaTemplate;
    
    public KafkaProducerService(KafkaTemplate<String, Object> kafkaTemplate) {
        this.kafkaTemplate = kafkaTemplate;
    }
    
    public void sendUserEvent(String topic, UserEvent userEvent) {
        CompletableFuture<SendResult<String, Object>> future = 
            kafkaTemplate.send(topic, userEvent.getUserId(), userEvent);
        
        future.whenComplete((result, ex) -> {
            if (ex == null) {
                log.info("消息发送成功: topic={}, key={}, partition={}, offset={}",
                        topic, userEvent.getUserId(), 
                        result.getRecordMetadata().partition(),
                        result.getRecordMetadata().offset());
            } else {
                log.error("消息发送失败: topic={}, key={}, error={}",
                        topic, userEvent.getUserId(), ex.getMessage());
            }
        });
    }
    
    public void sendWithCallback(String topic, String key, String message) {
        CompletableFuture<SendResult<String, Object>> future = 
            kafkaTemplate.send(topic, key, message);
        
        future.whenComplete((result, ex) -> {
            if (ex == null) {
                log.info("简单消息发送成功: topic={}, key={}, message={}",
                        topic, key, message);
            } else {
                log.error("简单消息发送失败: topic={}, key={}", topic, key, ex);
            }
        });
    }
}

3.7 消息消费者服务

KafkaConsumerService.java

package com.example.redpandademo.service;

import com.example.redpandademo.events.UserEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.messaging.handler.annotation.Header;
import org.springframework.messaging.handler.annotation.Payload;
import org.springframework.stereotype.Service;

@Service
public class KafkaConsumerService {
    
    private static final Logger log = LoggerFactory.getLogger(KafkaConsumerService.class);
    
    @KafkaListener(topics = "user-events", groupId = "user-group")
    public void consumeUserEvent(@Payload UserEvent userEvent,
                                @Header(KafkaHeaders.RECEIVED_KEY) String key,
                                @Header(KafkaHeaders.RECEIVED_PARTITION) int partition,
                                @Header(KafkaHeaders.OFFSET) long offset) {
        log.info("收到用户事件: key={}, partition={}, offset={}, event={}",
                key, partition, offset, userEvent);
        
        processUserEvent(userEvent);
    }
    
    @KafkaListener(topics = "order-events", groupId = "order-group")
    public void consumeOrderEvent(String message) {
        log.info("收到订单事件: {}", message);
        processOrderEvent(message);
    }
    
    private void processUserEvent(UserEvent userEvent) {
        log.info("处理用户事件: {}", userEvent.getEventType());
        
        switch (userEvent.getEventType()) {
            case "REGISTER":
                log.info("新用户注册: {}", userEvent.getEmail());
                break;
            case "LOGIN":
                log.info("用户登录: {}", userEvent.getEmail());
                break;
            default:
                log.info("未知用户事件类型: {}", userEvent.getEventType());
        }
    }
    
    private void processOrderEvent(String message) {
        log.info("处理订单事件: {}", message);
    }
}

3.8 REST控制器

DemoController.java

package com.example.redpandademo.controller;

import com.example.redpandademo.events.UserEvent;
import com.example.redpandademo.service.KafkaProducerService;
import org.springframework.web.bind.annotation.*;

@RestController
@RequestMapping("/api/kafka")
public class DemoController {
    
    private final KafkaProducerService kafkaProducerService;
    
    public DemoController(KafkaProducerService kafkaProducerService) {
        this.kafkaProducerService = kafkaProducerService;
    }
    
    @PostMapping("/user-event")
    public String sendUserEvent(@RequestParam String userId,
                               @RequestParam String eventType,
                               @RequestParam String email) {
        UserEvent userEvent = new UserEvent(userId, eventType, email);
        kafkaProducerService.sendUserEvent("user-events", userEvent);
        return "用户事件发送成功";
    }
    
    @PostMapping("/message")
    public String sendMessage(@RequestParam String topic,
                             @RequestParam String key,
                             @RequestParam String message) {
        kafkaProducerService.sendWithCallback(topic, key, message);
        return "消息发送成功";
    }
    
    @GetMapping("/health")
    public String health() {
        return "Spring Boot 3 + Redpanda 服务运行正常";
    }
}

3.9 主应用类

RedpandaDemoApplication.java

package com.example.redpandademo;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class RedpandaDemoApplication {
    
    public static void main(String[] args) {
        SpringApplication.run(RedpandaDemoApplication.class, args);
    }
}