spring boot对接kafka实现消息队列处理

一:依赖安装

复制代码
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-kafka</artifactId>
</dependency>
<!-- jackson 用于JSON序列化 -->
<dependency>
    <groupId>com.fasterxml.jackson.core</groupId>
    <artifactId>jackson-databind</artifactId>
</dependency>

二:kafka对接配置

复制代码
spring:
  kafka:
    # Kafka集群地址,多个节点逗号分隔:ip1:9092,ip2:9092
    bootstrap-servers: 127.0.0.1:9092
    # ===================== 生产者配置(发消息) =====================
    producer:
      # key序列化器:消息key序列化为字符串
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      # 生产者已经用 ObjectMapper 转成 JSON 字符串,这里必须用 StringSerializer,不能再用 JsonSerializer(会二次序列化)
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
      # acks确认机制:
      # 0:不需要broker确认,最快,丢消息风险高
      # 1:leader分区写入成功就返回(默认)
      # all:leader + 所有follower副本都同步完成才返回,可靠性最高,生产重要消息推荐
      acks: all
      # 发送失败最大重试次数(注意:重试会带来消息重复,业务必须幂等)
      retries: 2
    # ===================== 消费者配置(收消息) =====================
    consumer:
      # 消费组ID,同组消费者分摊分区;不同消费组独立消费全量消息
      group-id: order-consumer-group
      # 当当前消费组在kafka没有offset记录时的策略:
      # earliest:从topic最开头第一条消息开始消费
      # latest:只消费配置生效之后新产生的消息(默认)
      auto-offset-reset: earliest
      # key反序列化器
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      # 是否自动提交offset
      # true:kafka客户端后台自动提交,容易出现业务没处理完但offset提交(丢消息)
      # false:关闭自动提交,由代码手动 ack.acknowledge() 提交offset,可控,生产批量消费首选
      enable-auto-commit: false
      #每次最多返回消息数量
      max-poll-records: 100
    # ===================== listener监听器配置(消费线程、拉取行为) =====================
    listener:
      # 消费模式:SINGLE=一次消费一条;BATCH=批量拉取,一次拿到一批消息列表
      type: BATCH
      ack-mode: MANUAL
      # poll超时时间(ms):消费者去broker拉消息,如果暂时没有消息,最多阻塞等待多久返回
      poll-timeout: 3000
      # 消费并发线程数;一个线程绑定若干分区,**并发数不能大于topic分区总数**
      # 比如topic一共3分区,concurrency=3,每个线程1个分区,达到最大并行度;大于3的线程会空闲
      concurrency: 3

三:Topic 配置(业务 Topic + DLT 死信 Topic)

复制代码
package com.example.demo.kafka.config;

import org.apache.kafka.clients.admin.NewTopic;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class KafkaTopicConfig {
    //业务topic
    public static final String TOPIC_ORDER = "topic-order";
    //死信topic
    public static final String TOPIC_ORDER_DLT = "topic-order-dlt";

    @Bean
    public NewTopic orderTopic()
    {
        // topic名,分区数,副本数
        return new NewTopic(TOPIC_ORDER, 3, (short) 1);
    }

    @Bean
    public NewTopic orderDltTopic() {
        // 分区数要和业务 topic 一致,否则 DLT 按原分区号投递会失败
        return new NewTopic(TOPIC_ORDER_DLT, 3, (short) 1);
    }
}

四:全局异常处理器(DLT死信核心)

复制代码
package com.example.demo.kafka.config;

import lombok.RequiredArgsConstructor;
import org.apache.kafka.common.TopicPartition;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.listener.CommonErrorHandler;
import org.springframework.kafka.listener.DeadLetterPublishingRecoverer;
import org.springframework.kafka.listener.DefaultErrorHandler;
import org.springframework.util.backoff.FixedBackOff;

@Configuration
@RequiredArgsConstructor
public class KafkaErrorConfig {

    private final KafkaTemplate<String, String> kafkaTemplate;

    @Bean
    public CommonErrorHandler batchErrorHandler()
    {
        // 重试耗尽后,消息投递到死信topic
        DeadLetterPublishingRecoverer recoverer = new DeadLetterPublishingRecoverer(
                kafkaTemplate,
                (record, ex) -> new TopicPartition(KafkaTopicConfig.TOPIC_ORDER_DLT, record.partition())
        );
        // 重试策略:间隔1s,最多重试2次;重试2次还失败 → DLT
        FixedBackOff backOff = new FixedBackOff(1000L, 2L);
        DefaultErrorHandler errorHandler = new DefaultErrorHandler(recoverer, backOff);
        // 失败记录进 DLT 后提交它的 offset,避免死信后又被重复拉取
        errorHandler.setCommitRecovered(true);
        errorHandler.addNotRetryableExceptions(IllegalArgumentException.class);
        return errorHandler;
    }
}

五:生产者service

复制代码
package com.example.demo.kafka.service;

import com.example.demo.kafka.config.KafkaTopicConfig;
import lombok.RequiredArgsConstructor;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;
import tools.jackson.core.JacksonException;
import tools.jackson.databind.ObjectMapper;

import java.util.Map;

@Service
@RequiredArgsConstructor
public class KafkaProducer {
    private final KafkaTemplate<String, String> kafkaTemplate;
    private final ObjectMapper objectMapper;

    /**
     * 发送消息
     * @param key kafka消息key
     * @param data 消息体map
     */
    public void sendMessage(String key, Map<String, Object> data)
    {
        try {
            String json = objectMapper.writeValueAsString(data);

            kafkaTemplate.send(
                    KafkaTopicConfig.TOPIC_ORDER,
                    key, // key,相同orderId进入同一分区,保证顺序
                    json
            ).whenComplete((result, ex) -> {
                if (ex != null) {
                    // 发送失败:落地日志、告警、补发
                    ex.printStackTrace();
                    return;
                }
                System.out.printf("发送成功, partition:%d, offset:%d%n",
                        result.getRecordMetadata().partition(),
                        result.getRecordMetadata().offset());
            });
        } catch (JacksonException e) {
            e.printStackTrace();
        }
    }
}

六:消费者service

复制代码
package com.example.demo.kafka.service;

import com.example.demo.kafka.config.KafkaTopicConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.listener.BatchListenerFailedException;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;
import tools.jackson.databind.ObjectMapper;
import tools.jackson.databind.node.ObjectNode;

import java.util.List;

@Slf4j
@Component
public class KafkaConsumer {
    private final ObjectMapper objectMapper;

    public KafkaConsumer(ObjectMapper objectMapper) {
        this.objectMapper = objectMapper;
    }

    @KafkaListener(
            topics = KafkaTopicConfig.TOPIC_ORDER,
            groupId = "${spring.kafka.consumer.group-id}"
    )
    public void batchConsume(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        log.info("批量拉取消息,条数:{}", records.size());
        for (int i = 0; i < records.size(); i++) {
            ConsumerRecord<String, String> record = records.get(i);
            try {
                ObjectNode jsonNode = objectMapper.readValue(record.value(), ObjectNode.class);

                //获取消费消息
                String orderId = jsonNode.get("orderId").asText();
                Long userId = jsonNode.get("userId").asLong();
                String subject = jsonNode.get("subject").asText();
                Integer amount = jsonNode.get("amount").asInt();

                log.info("消费消息 orderId={}, userId={}, subject={}, amount={}",
                        orderId, userId, subject, amount);

                // ================= 你的业务代码在这里 =================
                if ("ERR-001".equals(orderId)) {
                    throw new RuntimeException("模拟业务处理失败");
                }
            } catch (Exception e) {
                log.error("批量第 {} 条失败, partition={}, offset={}", i, record.partition(), record.offset(), e);
                // 只把失败那条交给 DefaultErrorHandler;前面已成功的会提交 offset,不会整批重放
                throw new BatchListenerFailedException("批量消费失败", e, i);
            }
        }
        ack.acknowledge();
    }
}

七:DLT死信消费者(兜底监听)

复制代码
package com.example.demo.kafka.service;

import com.example.demo.kafka.config.KafkaTopicConfig;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;
import org.springframework.stereotype.Component;

import java.util.List;

@Slf4j
@Component
public class KafkaDltConsumer {
    // 全局 listener.type=BATCH 且 ack-mode=MANUAL,死信监听也必须是 List + 手动 ack
    @KafkaListener(topics = KafkaTopicConfig.TOPIC_ORDER_DLT, groupId = "order-dlt-group")
    public void listenDlt(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        for (ConsumerRecord<String, String> record : records) {
            log.error("===== 收到死信消息 =====");
            log.error("topic:{} , key:{} , value:{}",
                    record.topic(), record.key(), record.value());
        }
        ack.acknowledge();
    }
}

八:实现示例

复制代码
package com.example.demo.controller;

import com.example.demo.kafka.service.KafkaProducer;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.HashMap;
import java.util.Map;

@RestController
@RequestMapping("/kafka")
@RequiredArgsConstructor
public class KafkaController {

    private final KafkaProducer producer;

    @GetMapping("/send")
    public String send(){
        Map<String, Object> data = new HashMap<>();

        //消息key
        String orderId = "xxx";
        data.put("orderId", orderId);
        //发送的消息
        data.put("userId", "用户ID");
        data.put("subject", "主题");
        data.put("amount", "金额");

        producer.sendMessage(orderId, data);
        return "发送成功 orderId:" + orderId;
    }

    @GetMapping("/sendErr")
    public String sendErr() {
        Map<String, Object> data = new HashMap<>();
        data.put("orderId", "ERR-001");
        data.put("userId", 10002L);
        data.put("subject", "异常测试");
        data.put("amount", 100);
        producer.sendMessage("ERR-001", data);
        return "已发送异常测试消息";
    }
}
相关推荐
码事漫谈14 小时前
以后人人都可以成为导演
后端
Resean022314 小时前
Spring 源码分析(二)配置类的解析与扫描
java·后端·spring
一只爱撸猫的程序猿14 小时前
五周、两个开源仓库、九次拦截之后:AI 时代程序员缺的到底是什么
人工智能·后端·开源
北城笑笑16 小时前
Vue 110 ,若依SSE流式接口限次空500排查与解决
前端·javascript·vue.js·后端·vue
ServBay17 小时前
Local harness 将退役,VS Code 本地开发要不要搬到云端?
后端·ai编程·visual studio code
ashiho17 小时前
【LeetCode hot 100】3. 无重复字符的最长子串 (三种解法)
java·数据结构·后端·算法·leetcode·滑动窗口
张一凡9318 小时前
你的 AGENTS.md,真的管到模型写代码了吗?
前端·后端·agent
00后程序员18 小时前
iOS 实时活动怎么开发,Windows 上用 KXBuild 和 ActivityKit 写灵动岛
后端·ios
程序员cxuan18 小时前
推荐 10 个 WorkBuddy 中的邪修用法
人工智能·后端·程序员
Bacon18 小时前
给项目做一次"全身 CT":Agent Skills 生态里的代码体检全链路
前端·人工智能·后端