Apache Kafka使用

1. 引入依赖

 <dependency>
       <groupId>org.apache.kafka</groupId>
       <artifactId>kafka-clients</artifactId>
       <version>3.0.1</version>
   </dependency>

2. yaml配置信息

commons:
  kafka:
    enabled: true   #是否开启kafka
    bootstrapServers: 127.0.0.1:9092
    producer:
      keySerializer: org.apache.kafka.common.serialization.StringSerializer
      valueSerializer: org.apache.kafka.common.serialization.StringSerializer
    consumer:
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      group-id: msg-group # 消费者组
      auto-offset-reset: earliest

3. 配置类

@Data
@Configuration
@ConfigurationProperties(prefix = "commons.kafka")
public class CommonsKafkaProperties {

    private String bootstrapServers;
    private String bootstrapServers111;
    private Producer producer;
    private Consumer consumer;

    @Data
    public static class Producer {
        private String keySerializer;
        private String valueSerializer;
        private Properties props = new Properties();
    }

    @Data
    public static class Consumer {
        private String keyDeserializer;
        private String valueDeserializer;
        private String autoOffsetReset;
        private String groupId;
        private Properties props = new Properties();
    }
}

4. kafka生产者消费者配置类

public class KafkaConfiguration {

    @Resource
    private CommonsKafkaProperties commonsKafkaProperties;

    @Bean
    @ConditionalOnProperty(value = "commons.kafka.enabled", havingValue = "true")
    public KafkaProducer<String,String> kafkaProducer() {
        Properties props = new Properties();
        props.put("bootstrap.servers", commonsKafkaProperties.getBootstrapServers());
        props.put("key.serializer", commonsKafkaProperties.getProducer().getKeySerializer());
        props.put("value.serializer", commonsKafkaProperties.getProducer().getValueSerializer());
        props.putAll(commonsKafkaProperties.getProducer().getProps());
        return new KafkaProducer<>(props);
    }

    @Bean
    @ConditionalOnProperty(value = "commons.kafka.enabled", havingValue = "true")
    public KafkaConsumer<String, String> kafkaConsumer() {
        Properties props = new Properties();
        props.put("bootstrap.servers", commonsKafkaProperties.getBootstrapServers());
        props.put("key.deserializer", commonsKafkaProperties.getConsumer().getKeyDeserializer());
        props.put("value.deserializer", commonsKafkaProperties.getConsumer().getValueDeserializer());
        props.put("group.id", commonsKafkaProperties.getConsumer().getGroupId());
        props.put("auto.offset.reset", "earliest");
        props.putAll(commonsKafkaProperties.getConsumer().getProps());
        return new KafkaConsumer<>(props);
    }
}

5. 发送消息

@Slf4j
@RestController
@RequestMapping("/kafkaMsg")
public class MessageController {

    @Resource
    private KafkaProducer<String, String> kafkaProducer;

    @GetMapping( "/send")
    public void sendMessage(@RequestParam String message) {
        ProducerRecord<String, String> record = new ProducerRecord<>("demoEvent", message);
        record.headers().add("event","TSP-W".getBytes());
        log.error("sendMessage demoEvent：{}", message);
        kafkaProducer.send(record);
    }
}

6. 消费者

6.1. 第一种方式

• 发送消息时，设置了消息头部信息；消费时，可根据头部信息消费数据

@Slf4j
@Service
public class ApacheKafkaListener {

    @Autowired
    private KafkaConsumer<String, String> kafkaConsumer;

    @PostConstruct
    public void startListening() {
        // 订阅主题
        kafkaConsumer.subscribe(Collections.singletonList("demoEvent"));

        // 启动一个线程来处理消息
        new Thread(() -> {
            try {
                while (true) {
                    // 轮询消息
                    ConsumerRecords<String, String> records = kafkaConsumer.poll(Duration.ofMillis(100));
                    // 处理消息
                    records.forEach(this::handleMessage);
                }
            } catch (Exception e) {
                log.error("handle message error", e);
            } finally {
                kafkaConsumer.close();
            }
        }).start();
    }

    private void handleMessage(ConsumerRecord<String, String> record) {
        // 解析自定义头部
        String eventHeader = null;
        if (record.headers() != null) {
            eventHeader = new String(record.headers().lastHeader("event").value());
        }
        if (ObjectUtils.nullSafeEquals(eventHeader, "TSP-W")) {
            // 解析消息体
            String key = record.key();
            String value = record.value();
            // 处理逻辑
            log.error("Received message: key = {}, value = {}, eventHeader = {}", key, value, eventHeader);
        }
    }
}

6.2. 第二种方式

• 将每个主题的消费者定义成独立的Bean，可增强代码的灵活性和可扩展性。这种做法不仅能针对各个主题进行精细化的配置，还能满足不同场景下的需求。

@Slf4j
@Configuration
public class ApacheKafkaConsumer {

    @Resource
    private CommonsKafkaProperties commonsKafkaProperties;

    @Bean
    @ConditionalOnProperty(value = "commons.kafka.enabled", havingValue = "false")
    public KafkaConsumer<String, String> handle() {
        Properties props = new Properties();
        props.put("key.deserializer", commonsKafkaProperties.getConsumer().getKeyDeserializer());
        props.put("value.deserializer", commonsKafkaProperties.getConsumer().getValueDeserializer());
        props.put("bootstrap.servers", commonsKafkaProperties.getBootstrapServers());
        props.put("group.id", commonsKafkaProperties.getConsumer().getGroupId());
        props.put("max.partition.fetch.bytes", 10485760);
        props.put("max.poll.interval.ms", 600000);
        props.put("io.kafka.record.handle.threadpool.coreSize", 5);
        props.put("io.kafka.record.handle.threadpool.maxSize", 10);
        props.put("io.kafka.record.handle.threadpool.queueSize", 5);
        props.put("io.kafka.client.name", "receiveDemoEvent");
        props.put("io.kafka.record.handle.threadpool.namePrefix", "DemoEvent");
        props.putAll(commonsKafkaProperties.getConsumer().getProps());
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Collections.singletonList("lowLoadEvent"));
        // 启动消费消息的线程
        new Thread(() -> {
            while (true) {
                try {
                    // 使用poll获取消息
                    ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(100));
                    for (ConsumerRecord<String, String> record : records) {
                        log.error("消费到消息: {}", record.value());
                    }
                } catch (Exception e) {
                    log.error("Error while consuming messages", e);
                }
            }
        }).start();
        return consumer;
    }
}