flink消费kafka数据,按照指定时间开始消费

kafka中根据时间戳开始消费数据

java 复制代码
package com.cindasc.rtasset.source;

import com.cindasc.rtasset.util.Constants;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
import org.apache.flink.kafka.shaded.org.apache.kafka.clients.consumer.OffsetResetStrategy;
import org.apache.flink.kafka.shaded.org.apache.kafka.common.TopicPartition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.nodes.CollectionNode;

import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;

/**
 * 支持按topic指定开始消费时间戳
 *
 * @author 86158
 */
public class KafkaOffsetsInitializer implements OffsetsInitializer  {

    private Logger logger = LoggerFactory.getLogger(KafkaOffsetsInitializer.class);


    private static final long serialVersionUID = 1L;
    /**
     * key:topic,value:开始消费时间戳
     */
    private Map<String, Long> topicStartingTimestamps;
    private ParameterTool parameters;

    /**
     * @param topicStartingTimestamps
     * @param parameters
     */
    public KafkaOffsetsInitializer(Map<String, Long> topicStartingTimestamps, ParameterTool parameters) {
        this.topicStartingTimestamps = topicStartingTimestamps;
        this.parameters = parameters;
    }

    @Override
    public Map<TopicPartition, Long> getPartitionOffsets(Collection<TopicPartition> partitions,
                                                         PartitionOffsetsRetriever partitionOffsetsRetriever) {
        //定义起始时间,初始offset
        Map<TopicPartition, Long> startingTimestamps = new HashMap<>();
        Map<TopicPartition, Long> initialOffsets = new HashMap<>();

        //commited offset
        Map<TopicPartition, Long> committedOffsets = partitionOffsetsRetriever.committedOffsets(partitions);

        //beginningOffsets the first offset for the given partitions.
        Map<TopicPartition, Long> beginningOffsets = partitionOffsetsRetriever.beginningOffsets(partitions);
        //endOffsets the for the given partitions.
        Map<TopicPartition, Long> endOffsets = partitionOffsetsRetriever.endOffsets(partitions);

        final long now = System.currentTimeMillis();
        partitions.forEach(tp -> {
            //起始时间赋值为从redis中获取到相对应topic的时间
            Long startingTimestamp = topicStartingTimestamps.get(tp.topic());
            if (startingTimestamp == null) {
                //redis里没有取到消费开始时间从启动时间消费
                startingTimestamp = now;
                logger.info("从redis没有取到时间戳,topic:{},partition:{},使用当前时间:{},{}", tp.topic(), tp.partition(), now, new Date(now));
            }
            logger.info("读取时间戳,topic:{},partition:{},时间戳:{},{}", tp.topic(), tp.partition(), now, new Date(now));
            startingTimestamps.put(tp, startingTimestamp);
        });
        partitionOffsetsRetriever.offsetsForTimes(startingTimestamps).forEach((tp, offsetMetadata) -> {
              long offsetForTime = beginningOffsets.get(tp);
            long offsetForCommit = beginningOffsets.get(tp);
            if (offsetMetadata != null) {
                offsetForTime = offsetMetadata.offset();
                logger.info("根据时间戳取到offset,topic:{},partition:{},offset:{}", tp.topic(), tp.partition(), offsetForTime);
            }

            Long commitedOffset = committedOffsets.get(tp);
            if (commitedOffset != null) {
                offsetForCommit = commitedOffset.longValue();
                logger.info("根据已提交offset取到offset,topic:{},partition:{},offset:{}", tp.topic(), tp.partition(), offsetForCommit);
            }
            logger.info("设置读取offset,topic:{},partition:{},offset:{},endOffset:{}", tp.topic(), tp.partition(), Math.max(offsetForTime, offsetForCommit), endOffsets.get(tp));
            //对比时间戳对应的offset和checkpoint保存的offset,取较大值
            //initialOffsets.put(tp, Math.max(offsetForTime, offsetForCommit));
            initialOffsets.put(tp, offsetForCommit);

            if (parameters.getBoolean(Constants.LOADERFAILED,false)){
                initialOffsets.put(tp, Math.min(offsetForTime, offsetForCommit));
            }
        });
        return initialOffsets;
    }

    @Override
    public OffsetResetStrategy getAutoOffsetResetStrategy() {
        return OffsetResetStrategy.NONE;
    }
}
相关推荐
你喜欢喝可乐吗?7 分钟前
RuoYi-Cloud 验证码处理流程
java·spring cloud·微服务·vue
Java技术小馆36 分钟前
langChain开发你的第一个 Agent
java·面试·架构
kangkang-37 分钟前
PC端基于SpringBoot架构控制无人机(二):MavLink协议
java·spring boot·后端·无人机
Dcs1 小时前
Anthropic 爆严重安全漏洞!程序员机器沦陷
java
EnigmaCoder1 小时前
Java多线程:核心技术与实战指南
java·开发语言
攀小黑1 小时前
阿里云 使用TST Token发送模板短信
java·阿里云
麦兜*1 小时前
Spring Boot秒级冷启动方案:阿里云FC落地实战(含成本对比)
java·spring boot·后端·spring·spring cloud·系统架构·maven
自由鬼2 小时前
正向代理服务器Squid:功能、架构、部署与应用深度解析
java·运维·服务器·程序人生·安全·架构·代理
fouryears_234172 小时前
深入拆解Spring核心思想之一:IoC
java·后端·spring
codervibe3 小时前
使用 Spring Boot + JWT 实现多角色登录认证(附完整流程图)
java·后端