Flink demo代码 - 技术栈

Flink流任务，用于checkpoint问题验证

java 复制代码

public class StreamDemo {

    public static void main(String[] args) throws Exception {
        // 1. 创建执行环境
//        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        StreamExecutionEnvironment env = StreamExecutionEnvironment
                .createLocalEnvironmentWithWebUI(new Configuration());

        // 2. 设置并行度
        env.setParallelism(1);

        // 3. 启用Checkpoint（使作业可以长期运行）
        env.enableCheckpointing(5000); // 5秒一次
        env.getCheckpointConfig().setCheckpointStorage("file:////Users/wangqin/IdeaProjects/MyFlinkCode/checkpoint");
        CheckpointConfig checkpointConfig = env.getCheckpointConfig();
        checkpointConfig.setExternalizedCheckpointCleanup(
                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION
        );

        env.getCheckpointConfig().setTolerableCheckpointFailureNumber(0);

        // 4. 创建无限数据源（一直产生数据）
        DataStream<String> infiniteStream = env.addSource(new InfiniteSource()).setParallelism(4);

        // 5. 简单的数据处理
        infiniteStream
                .map(value -> "处理数据: " + value + ", 时间: " + System.currentTimeMillis())
                .print();

        // 6. 执行作业（会一直运行）
        env.execute("Infinite Stream Job");
    }

    /**
     * 自定义无限数据源
     * 每秒生成一个随机数，一直运行
     */
    public static class InfiniteSource extends RichParallelSourceFunction<String> implements CheckpointedFunction {
        private volatile boolean isRunning = true;
        private long count = 0;

        @Override
        public void run(SourceContext<String> ctx) throws Exception {
            while (isRunning) {
                // 每秒生成一个数据
                String data = "随机数-" + (int)(Math.random() * 100) + "-计数-" + (count++);
                ctx.collect(data);

                // 控制数据生成速度：每秒1条
                Thread.sleep(1000);

                // 每10条数据打印一次日志
                if (count % 10 == 0) {
                    System.out.println("[Source] 已生成 " + count + " 条数据，继续运行中...");
                }
            }
        }

        @Override
        public void cancel() {
            isRunning = false;
            System.out.println("数据源已停止");
        }

        @Override
        public void snapshotState(FunctionSnapshotContext functionSnapshotContext) throws Exception {
            int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
            System.out.println("subtaskIndex is" + subtaskIndex);
//            if (subtaskIndex == 0) {
//                Thread.sleep(10000);
//                throw new RuntimeException("checkpoint failed");
//            }
            if (functionSnapshotContext.getCheckpointId() % 3 == 0) {
                throw new RuntimeException("checkpoint failed");
            }
        }

        @Override
        public void initializeState(FunctionInitializationContext functionInitializationContext) throws Exception {

        }
    }
}

本地运行Flink任务发现，在snapshotState方法中抛出异常，不会生成checkpoint目录及metadata file；延长方法的执行时间，在执行checkpoint的时候会先生成空的checkpoint目录。

checkpoint metadata file解析

java 复制代码

public class CheckpointMetaDataAnalyzer {
    public static void main(String[] args) throws Exception {
        File metadataFile = new File("./checkpoint/d3299c5896dee8c5870606e41b0e0c0a/chk-5", "_metadata");
        try (DataInputStream dis = new DataInputStream(Files.newInputStream(metadataFile.toPath()))) {
            ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
            CheckpointMetadata metadata = Checkpoints.loadCheckpointMetadata(
                    dis, classLoader, "./checkpoint/d3299c5896dee8c5870606e41b0e0c0a/chk-5");
            System.out.println("Loaded checkpoint ID: " + metadata.getCheckpointId());
        }
    }
}