springboot集成kafka消费数据

springboot集成kafka消费数据

文章目录

1.引入pom依赖

xml 复制代码
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
            <version>2.8.11</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>3.1.2</version>
        </dependency>

2.添加配置文件

2.1.添加KafkaConsumerConfig.java

java 复制代码
@Configuration
@EnableConfigurationProperties(KafkaIotCustomProperties.class)
@Slf4j
public class KafkaConsumerConfig {

    @Autowired
    KafkaIotCustomProperties kafkaIotCustomProperties;


    @Bean
    KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactory());
        // 并发数 多个微服务实例会均分
        factory.setConcurrency(3);
        factory.setBatchListener(true);
        ContainerProperties containerProperties = factory.getContainerProperties();
        // 是否设置手动提交
        containerProperties.setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
        return factory;
    }

    private ConsumerFactory<String, String> consumerFactory() {
        Map<String, Object> consumerConfigs = consumerConfigs();
        log.info("消费者的配置信息:{}",JSONObject.toJSONString(consumerConfigs));
        return new DefaultKafkaConsumerFactory<>(consumerConfigs);
    }


    @Bean
    public Map<String, Object> consumerConfigs() {
        Map<String, Object> propsMap = new HashMap<>();
        // 服务器地址
        propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaIotCustomProperties.getBootstrapServers());
        // 是否自动提交
        propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, kafkaIotCustomProperties.isEnableAutoCommit());
        // 自动提交间隔
        propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, kafkaIotCustomProperties.getAutoCommitInterval());
        //会话时间
        propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, kafkaIotCustomProperties.getSessionTimeOut());
        //key序列化
        propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, kafkaIotCustomProperties.getKeyDeserializer());
        //value序列化
        propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, kafkaIotCustomProperties.getValueDeserializer());
        // 心跳时间
        propsMap.put(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, kafkaIotCustomProperties.getHeartbeatInterval());

        // 分组id
        propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, kafkaIotCustomProperties.getGroupId());
        //消费策略
        propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, kafkaIotCustomProperties.getAutoOffsetReset());
        // poll记录数
        propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, kafkaIotCustomProperties.getMaxPollRecords());
        //poll时间
        propsMap.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, kafkaIotCustomProperties.getMaxPollInterval());
        return propsMap;
    }

}

2.2.添加KafkaIotCustomProperties.java

java 复制代码
@Component
@ConfigurationProperties(prefix = "fxyh.realdata.kafka")
@Data
public class KafkaIotCustomProperties {

    private List<String> topics;

    private String groupId;

    private String sessionTimeOut;

    private String bootstrapServers;

    private String autoOffsetReset;

    private boolean enableAutoCommit;

    private String autoCommitInterval;

    private String fetchMinSize;

    private String fetchMaxWait;

    private String maxPollRecords;

    private String maxPollInterval;

    private String heartbeatInterval;

    private String keyDeserializer;

    private String valueDeserializer;
}

2.3.添加application.yml配置

yaml 复制代码
fxyh:
  realdata:
    kafka:
      bootstrapServers:  192.168.80.251:9092
      topics: ["test1","test2"]
      groupId: shengtingrealdatagroup
      #后台的心跳线程必须在30秒之内提交心跳,否则会reBalance
      sessionTimeOut: 30000
      #      autoOffsetReset: earliest
      #取消自动提交,即便如此 spring会帮助我们自动提交
      enableAutoCommit: false
      #自动提交间隔
      autoCommitInterval: 1000
      #拉取的最小字节
      fetchMinSize: 1
      #拉去最小字节的最大等待时间
      fetchMaxWait: 500
      maxPollRecords: 50
      #300秒的提交间隔,如果程序大于300秒提交,会报错
      maxPollInterval: 300000
      #心跳间隔
      heartbeatInterval: 10000
      keyDeserializer: org.apache.kafka.common.serialization.StringDeserializer
      valueDeserializer: org.apache.kafka.common.serialization.StringDeserializer
      auto-offset-reset: latest

3.消费者代码

java 复制代码
@Slf4j
@Component
public class DeviceDataConsumer {


    @Autowired
    private KafkaIotCustomProperties kafkaIotCustomProperties;


    @KafkaListener(topics = {"#{@kafkaIotCustomProperties.topics}"}, groupId = "#{@kafkaIotCustomProperties.groupId}", containerFactory = "kafkaListenerContainerFactory",properties = {"#{@kafkaIotCustomProperties.autoOffsetReset}"})
    public void topicTest(List<ConsumerRecord<String, String>> records, Acknowledgment ack) {
        for (ConsumerRecord<String, String> record : records) {
            log.info("topic_test 消费了: Topic:" + record.topic() + ",groupId:" + kafkaIotCustomProperties.getGroupId() + ",Message:" + record.value());
            //手动提交偏移量
            ack.acknowledge();

        }
    }
}
相关推荐
特立独行的猫A7 小时前
别再用 nohup 了:让应用在后台常驻的正确姿势与 systemd 服务详解
java·后端·python
一起_java7 小时前
技术文章-基于Dify-Agent的智能问答系统设计与实现
java·spring boot·后端·agent·dify
Flynn Liang9 小时前
I/O 密集型高并发实战:Java 服务 10 维优化,从线程打满到 10 万 QPS
java·spring boot·性能优化·高并发·系统设计·虚拟线程·io密集型
吃我一窝jio10 小时前
Python与Go线程模型深度对比
java·开发语言·python
这个昵称不容易10 小时前
SpringBoot 购物推荐网站的设计与实现:120 个 Java 文件、147 个接口、13 张表,附真实运行截图
java·springboot·毕设
文丑颜不良啊10 小时前
LeetCode3041~3060题目解析
java·leetcode
文丑颜不良啊12 小时前
LeetCode3101~3120题目解析
java·leetcode
Resean022312 小时前
SpringMVC 6+源码分析(十一)视图渲染与异步请求(收尾篇)
java·spring·springmvc
靠沿12 小时前
MySQL数据库进阶——存储过程和触发器
java·数据库·mysql
码少女13 小时前
MySQL--库与表的增删改查基础
java·数据库·mysql