spring boot对接kafka实现消息队列处理

一：依赖安装

复制代码

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-kafka</artifactId>
</dependency>
<!-- jackson 用于JSON序列化 -->
<dependency>
    <groupId>com.fasterxml.jackson.core</groupId>
    <artifactId>jackson-databind</artifactId>
</dependency>

二：kafka对接配置

复制代码

spring:
  kafka:
    # Kafka集群地址，多个节点逗号分隔：ip1:9092,ip2:9092
    bootstrap-servers: 127.0.0.1:9092
    # ===================== 生产者配置（发消息） =====================
    producer:
      # key序列化器：消息key序列化为字符串
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      # 生产者已经用 ObjectMapper 转成 JSON 字符串，这里必须用 StringSerializer，不能再用 JsonSerializer（会二次序列化）
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
      # acks确认机制：
      # 0：不需要broker确认，最快，丢消息风险高
      # 1：leader分区写入成功就返回（默认）
      # all：leader + 所有follower副本都同步完成才返回，可靠性最高，生产重要消息推荐
      acks: all
      # 发送失败最大重试次数（注意：重试会带来消息重复，业务必须幂等）
      retries: 2
    # ===================== 消费者配置（收消息） =====================
    consumer:
      # 消费组ID，同组消费者分摊分区；不同消费组独立消费全量消息
      group-id: order-consumer-group
      # 当当前消费组在kafka没有offset记录时的策略：
      # earliest：从topic最开头第一条消息开始消费
      # latest：只消费配置生效之后新产生的消息（默认）
      auto-offset-reset: earliest
      # key反序列化器
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      # 是否自动提交offset
      # true：kafka客户端后台自动提交，容易出现业务没处理完但offset提交（丢消息）
      # false：关闭自动提交，由代码手动 ack.acknowledge() 提交offset，可控，生产批量消费首选
      enable-auto-commit: false
      #每次最多返回消息数量
      max-poll-records: 100
    # ===================== listener监听器配置（消费线程、拉取行为） =====================
    listener:
      # 消费模式：SINGLE=一次消费一条；BATCH=批量拉取，一次拿到一批消息列表
      type: BATCH
      ack-mode: MANUAL
      # poll超时时间(ms)：消费者去broker拉消息，如果暂时没有消息，最多阻塞等待多久返回
      poll-timeout: 3000
      # 消费并发线程数；一个线程绑定若干分区，**并发数不能大于topic分区总数**
      # 比如topic一共3分区，concurrency=3，每个线程1个分区，达到最大并行度；大于3的线程会空闲
      concurrency: 3

三：Topic 配置（业务 Topic + DLT 死信 Topic）

复制代码

package com.example.demo.kafka.config;

import org.apache.kafka.clients.admin.NewTopic;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class KafkaTopicConfig {
    //业务topic
    public static final String TOPIC_ORDER = "topic-order";
    //死信topic
    public static final String TOPIC_ORDER_DLT = "topic-order-dlt";

    @Bean
    public NewTopic orderTopic()
    {
        // topic名，分区数，副本数
        return new NewTopic(TOPIC_ORDER, 3, (short) 1);
    }

    @Bean
    public NewTopic orderDltTopic() {
        // 分区数要和业务 topic 一致，否则 DLT 按原分区号投递会失败
        return new NewTopic(TOPIC_ORDER_DLT, 3, (short) 1);
    }
}

四：全局异常处理器（DLT死信核心）

复制代码

package com.example.demo.kafka.config;

import lombok.RequiredArgsConstructor;
import org.apache.kafka.common.TopicPartition;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.listener.CommonErrorHandler;
import org.springframework.kafka.listener.DeadLetterPublishingRecoverer;
import org.springframework.kafka.listener.DefaultErrorHandler;
import org.springframework.util.backoff.FixedBackOff;

@Configuration
@RequiredArgsConstructor
public class KafkaErrorConfig {

    private final KafkaTemplate<String, String> kafkaTemplate;

    @Bean
    public CommonErrorHandler batchErrorHandler()
    {
        // 重试耗尽后，消息投递到死信topic
        DeadLetterPublishingRecoverer recoverer = new DeadLetterPublishingRecoverer(
                kafkaTemplate,
                (record, ex) -> new TopicPartition(KafkaTopicConfig.TOPIC_ORDER_DLT, record.partition())
        );
        // 重试策略：间隔1s，最多重试2次；重试2次还失败 → DLT
        FixedBackOff backOff = new FixedBackOff(1000L, 2L);
        DefaultErrorHandler errorHandler = new DefaultErrorHandler(recoverer, backOff);
        // 失败记录进 DLT 后提交它的 offset，避免死信后又被重复拉取
        errorHandler.setCommitRecovered(true);
        errorHandler.addNotRetryableExceptions(IllegalArgumentException.class);
        return errorHandler;
    }
}

五：生产者service

复制代码

package com.example.demo.kafka.service;

import com.example.demo.kafka.config.KafkaTopicConfig;
import lombok.RequiredArgsConstructor;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.ObjectMapper;

import java.util.Map;

@Service
@RequiredArgsConstructor
public class KafkaProducer {
    private final KafkaTemplate<String, String> kafkaTemplate;
    private final ObjectMapper objectMapper;

    /**
     * 发送消息
     * @param key kafka消息key
     * @param data 消息体map
     */
    public void sendMessage(String key, Map<String, Object> data)
    {
        try {
            String json = objectMapper.writeValueAsString(data);

            kafkaTemplate.send(
                    KafkaTopicConfig.TOPIC_ORDER,
                    key, // key，相同orderId进入同一分区，保证顺序
                    json
            ).whenComplete((result, ex) -> {
                if (ex != null) {
                    // 发送失败：落地日志、告警、补发
                    ex.printStackTrace();
                    return;
                }
                System.out.printf("发送成功, partition:%d, offset:%d%n",
                        result.getRecordMetadata().partition(),
                        result.getRecordMetadata().offset());
            });
        } catch (JacksonException e) {
            e.printStackTrace();
        }
    }
}

六：消费者service

复制代码

package com.example.demo.kafka.service;

import com.example.demo.kafka.config.KafkaTopicConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.listener.BatchListenerFailedException;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.node.ObjectNode;

import java.util.List;

@Slf4j
@Component
public class KafkaConsumer {
    private final ObjectMapper objectMapper;

    public KafkaConsumer(ObjectMapper objectMapper) {
        this.objectMapper = objectMapper;
    }

    @KafkaListener(
            topics = KafkaTopicConfig.TOPIC_ORDER,
            groupId = "${spring.kafka.consumer.group-id}"
    )
    public void batchConsume(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        log.info("批量拉取消息，条数：{}", records.size());
        for (int i = 0; i < records.size(); i++) {
            ConsumerRecord<String, String> record = records.get(i);
            try {
                ObjectNode jsonNode = objectMapper.readValue(record.value(), ObjectNode.class);

                //获取消费消息
                String orderId = jsonNode.get("orderId").asText();
                Long userId = jsonNode.get("userId").asLong();
                String subject = jsonNode.get("subject").asText();
                Integer amount = jsonNode.get("amount").asInt();

                log.info("消费消息 orderId={}, userId={}, subject={}, amount={}",
                        orderId, userId, subject, amount);

                // ================= 你的业务代码在这里 =================
                if ("ERR-001".equals(orderId)) {
                    throw new RuntimeException("模拟业务处理失败");
                }
            } catch (Exception e) {
                log.error("批量第 {} 条失败, partition={}, offset={}", i, record.partition(), record.offset(), e);
                // 只把失败那条交给 DefaultErrorHandler；前面已成功的会提交 offset，不会整批重放
                throw new BatchListenerFailedException("批量消费失败", e, i);
            }
        }
        ack.acknowledge();
    }
}

七：DLT死信消费者（兜底监听）

复制代码

package com.example.demo.kafka.service;

import com.example.demo.kafka.config.KafkaTopicConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;

import java.util.List;

@Slf4j
@Component
public class KafkaDltConsumer {
    // 全局 listener.type=BATCH 且 ack-mode=MANUAL，死信监听也必须是 List + 手动 ack
    @KafkaListener(topics = KafkaTopicConfig.TOPIC_ORDER_DLT, groupId = "order-dlt-group")
    public void listenDlt(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        for (ConsumerRecord<String, String> record : records) {
            log.error("===== 收到死信消息 =====");
            log.error("topic:{} , key:{} , value:{}",
                    record.topic(), record.key(), record.value());
        }
        ack.acknowledge();
    }
}

八：实现示例

复制代码

package com.example.demo.controller;

import com.example.demo.kafka.service.KafkaProducer;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.HashMap;
import java.util.Map;

@RestController
@RequestMapping("/kafka")
@RequiredArgsConstructor
public class KafkaController {

    private final KafkaProducer producer;

    @GetMapping("/send")
    public String send(){
        Map<String, Object> data = new HashMap<>();

        //消息key
        String orderId = "xxx";
        data.put("orderId", orderId);
        //发送的消息
        data.put("userId", "用户ID");
        data.put("subject", "主题");
        data.put("amount", "金额");

        producer.sendMessage(orderId, data);
        return "发送成功 orderId:" + orderId;
    }

    @GetMapping("/sendErr")
    public String sendErr() {
        Map<String, Object> data = new HashMap<>();
        data.put("orderId", "ERR-001");
        data.put("userId", 10002L);
        data.put("subject", "异常测试");
        data.put("amount", 100);
        producer.sendMessage("ERR-001", data);
        return "已发送异常测试消息";
    }
}