FlinkSQL实现实时同步和实时统计过程（MySQL TO MySQL）

Eugene Jou2025-03-16 13:46

实时同步

注意mysql表的主键要和FlinkSQL的一致

sql 复制代码 
set execution.checkpointing.checkpoints - after - tasks - finish.enabled = true;
SET pipeline.operator - chaining = false;
set state.backend.type = rocksdb;
set execution.checkpointing.interval = 8000;
set state.checkpoints.num - retained = 10;
set cluster.evenly - spread - out - slots = true;

CREATE TABLE IF NOT EXISTS study_score_cdc (
  cid INT,
  sid INT,
  cls STRING,
  score INT,
  PRIMARY KEY (cid) NOT ENFORCED
)
WITH
  (
    'connector' = 'mysql-cdc',
    'hostname' = '192.168.20.23',
    'port' = '3306',
    'username' = 'root',
    'password' = 'ccphl123!',
    'database-name' = 'db_eugene_test',
    'server-time-zone' = 'Asia/Shanghai',
    --  'server-time-zone' = 'UTC',
    --  'scan.incremental.snapshot.enabled' = 'true',
    --  'debezium.snapshot.mode' = 'initial', -- 或者key是scan.startup.mode，initial表示要历史数据，latest-offset表示不要历史数据
    --  'debezium.datetime.format.date' = 'yyyy-MM-dd',
    --  'debezium.datetime.format.time' = 'HH-mm-ss',
    --  'debezium.datetime.format.datetime' = 'yyyy-MM-dd HH-mm-ss',
    --  'debezium.datetime.format.timestamp' = 'yyyy-MM-dd HH-mm-ss',
    --  'debezium.datetime.format.timestamp.zone' = 'UTC+8',
    'table-name' = 'study_score'
  );

DROP table IF EXISTS study_score_cdc_sink;

CREATE TABLE IF NOT EXISTS study_score_cdc_sink (
  cid INT,
  sid INT,
  cls STRING,
  score INT,
  PRIMARY KEY (cid) NOT ENFORCED
)
WITH
  (
    'connector' = 'jdbc',
    'url' = 'jdbc:mysql://192.168.21.201:3308/db_eugene_test_01?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2B8',
    'username' = 'root',
    'password' = 'redrcd@123',
    'table-name' = 'study_score_cdc_sink'
  );

INSERT INTO
  study_score_cdc_sink 
select
  cid,
  sid,
  cls,
  score
from
  study_score_cdc;

实时统计（无历史记录）

mysql表：

sql 复制代码 
CREATE TABLE `score_statistics` (
  `cls` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
  `score` int(11) DEFAULT NULL,
  PRIMARY KEY (`cls`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

注意mysql表的主键要和FlinkSQL的一致

sql 复制代码 
-- 配置参数（原有配置保留）
SET execution.checkpointing.checkpoints-after-tasks-finish.enabled = true;
SET pipeline.operator-chaining = false;
SET state.backend.type = rocksdb;
SET execution.checkpointing.interval = 8000;
SET state.checkpoints.num-retained = 10;
SET cluster.evenly-spread-out-slots = true;
SET execution.time-characteristic = 'ProcessingTime';
SET table.exec.timezone = 'Asia/Shanghai';

-- CDC 源表（保持不变）
CREATE TABLE IF NOT EXISTS study_score_cdc (
    cid INT,
    sid INT,
    cls STRING,
    score INT,
    PRIMARY KEY (cid) NOT ENFORCED
) WITH (
    'connector' = 'mysql-cdc',
    'hostname' = '192.168.20.23',
    'port' = '3306',
    'username' = 'root',
    'password' = 'ccphl123!',
    'database-name' = 'db_eugene_test',
    'server-time-zone' = 'Asia/Shanghai',
    'table-name' = 'study_score'
);

-- Sink 表（所有时间字段为 STRING 类型）
CREATE TABLE IF NOT EXISTS study_score_cdc_statistics_sink (
    cls STRING PRIMARY KEY NOT ENFORCED,  -- 主键为业务键 cls，和MySQL的
    score INT
) WITH (
    'connector' = 'jdbc',
    'url' = 'jdbc:mysql://192.168.21.201:3308/db_eugene_test_01?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2B8',
    'username' = 'root',
    'password' = 'redrcd@123',
    'table-name' = 'score_statistics',
    'sink.buffer-flush.max-rows' = '1'  -- 控制写入频率
);

-- 执行更新逻辑
INSERT INTO study_score_cdc_statistics_sink 
WITH
-- 计算最新聚合数据
newa_records AS (
    SELECT 
        cls,
        SUM(score) AS score
    FROM study_score_cdc
    GROUP BY cls
)
-- 合并新旧数据并执行 UPSERT
SELECT 
    cls,
    score
FROM newa_records;

拉链表

注意目标表的联合主键和FlinkSQL的保持一致

sql 复制代码 
CREATE TABLE `dim_score_statistics` (
  `id` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
  `cls` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
  `score` int(11) DEFAULT NULL,
  `update_time` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `dw_start_time` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
  `dw_end_time` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
  `is_current` tinyint(1) NOT NULL,
  -- 这儿重点注意以下
  PRIMARY KEY (`cls`,`dw_end_time`,`is_current`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

实时统计（有历史记录，每次运行如果目标表有数据都会变成历史数据）

sql 复制代码 
-- 配置参数（原有配置保留）
SET execution.checkpointing.checkpoints-after-tasks-finish.enabled = true;
SET pipeline.operator-chaining = false;
SET state.backend.type = rocksdb;
SET execution.checkpointing.interval = 8000;
SET state.checkpoints.num-retained = 10;
SET cluster.evenly-spread-out-slots = true;
SET execution.time-characteristic = 'ProcessingTime';
SET table.exec.timezone = 'Asia/Shanghai';

-- CDC 源表（保持不变）
CREATE TABLE IF NOT EXISTS study_score_cdc (
    cid INT,
    sid INT,
    cls STRING,
    score INT,
    PRIMARY KEY (cid) NOT ENFORCED
) WITH (
    'connector' = 'mysql-cdc',
    'hostname' = '192.168.20.23',
    'port' = '3306',
    'username' = 'root',
    'password' = 'ccphl123!',
    'database-name' = 'db_eugene_test',
    'server-time-zone' = 'Asia/Shanghai',
    'table-name' = 'study_score'
);

-- Sink 表（根据 MySQL 主键调整字段顺序）
CREATE TABLE IF NOT EXISTS dim_score_statistics_sink (
    cls STRING,
    dw_end_time STRING,
    is_current BOOLEAN,
    id STRING,
    score INT,
    update_time STRING,
    dw_start_time STRING,
    -- 主键必须与 MySQL 的组合主键一致
    PRIMARY KEY (cls, dw_end_time, is_current) NOT ENFORCED
) WITH (
    'connector' = 'jdbc',
    'url' = 'jdbc:mysql://192.168.21.201:3308/db_eugene_test_01?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2B8',
    'username' = 'root',
    'password' = 'redrcd@123',
    'table-name' = 'dim_score_statistics',
    'sink.buffer-flush.max-rows' = '1'  -- 控制写入频率
);

-- 执行更新逻辑
INSERT INTO dim_score_statistics_sink 
WITH
-- 新记录（当前有效）
newa_records AS (
    SELECT 
        cls,
        '9999-12-31 23:59:59' AS dw_end_time,
        TRUE AS is_current,
        UUID() AS id,
        SUM(score) AS score,
        DATE_FORMAT(CURRENT_TIMESTAMP, 'yyyy-MM-dd HH:mm:ss') AS update_time,
        DATE_FORMAT(CURRENT_TIMESTAMP, 'yyyy-MM-dd HH:mm:ss') AS dw_start_time
    FROM study_score_cdc 
    GROUP BY cls
),
-- 获取所有当前有效旧记录
olda_records AS (
    SELECT 
        cls,
        dw_end_time,
        is_current,
        id,
        score,
        update_time,
        dw_start_time
    FROM dim_score_statistics_sink
    WHERE is_current = TRUE
),
-- 生成失效的旧记录（主键组合为 (cls, 当前时间, FALSE)）
closed_olda_records AS (
    SELECT 
        olda.cls AS cls,
        DATE_FORMAT(CURRENT_TIMESTAMP, 'yyyy-MM-dd HH:mm:ss') AS dw_end_time,  -- 过期时间设为当前时间
        FALSE AS is_current,
        olda.id AS id,
        olda.score AS score,
        olda.update_time AS update_time,
        olda.dw_start_time AS dw_start_time
    FROM olda_records AS olda
)
-- 合并新记录和失效的旧记录
SELECT 
    cls,
    dw_end_time,
    is_current,
    id,
    score,
    update_time,
    dw_start_time
FROM newa_records
UNION ALL
SELECT 
    cls,
    dw_end_time,
    is_current,
    id,
    score,
    update_time,
    dw_start_time
FROM closed_olda_records;

实时统计（有历史记录，只新增score有变动的）

sql 复制代码 
-- 配置参数（原有配置保留）
SET execution.checkpointing.checkpoints-after-tasks-finish.enabled = true;
SET pipeline.operator-chaining = false;
SET state.backend.type = rocksdb;
SET execution.checkpointing.interval = 8000;
SET state.checkpoints.num-retained = 10;
SET cluster.evenly-spread-out-slots = true;
SET execution.time-characteristic = 'ProcessingTime';
SET table.exec.timezone = 'Asia/Shanghai';

-- CDC 源表（保持不变）
CREATE TABLE IF NOT EXISTS study_score_cdc   (
    cid INT,
    sid INT,
    cls STRING,
    score INT,
    PRIMARY KEY (cid) NOT ENFORCED
) WITH (
    'connector' = 'mysql-cdc',
    'hostname' = '192.168.20.23',
    'port' = '3306',
    'username' = 'root',
    'password' = 'ccphl123!',
    'database-name' = 'db_eugene_test',
    'server-time-zone' = 'Asia/Shanghai',
    'table-name' = 'study_score'
);

-- Sink 表（主键为 (cls, dw_end_time, is_current)）
CREATE TABLE IF NOT EXISTS dim_score_statistics_sink (
    cls STRING,
    dw_end_time STRING,
    is_current BOOLEAN,
    id STRING,
    score INT,
    update_time STRING,
    dw_start_time STRING,
    PRIMARY KEY (cls, dw_end_time, is_current) NOT ENFORCED
) WITH (
    'connector' = 'jdbc',
    'url' = 'jdbc:mysql://192.168.21.201:3308/db_eugene_test_01?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2B8',
    'username' = 'root',
    'password' = 'redrcd@123',
    'table-name' = 'dim_score_statistics',
    'sink.buffer-flush.max-rows' = '1'
);

-- 执行更新逻辑
INSERT INTO dim_score_statistics_sink
WITH
-- 计算新记录的 score
newa_score AS (
    SELECT 
        cls,
        SUM(score) AS new_score
    FROM study_score_cdc  
    GROUP BY cls
),
-- 获取当前有效旧记录的字段（包含所有字段）
olda_current AS (
    SELECT 
        cls,
        score AS old_score,
        dw_start_time,
        dw_end_time,
        is_current,
        id,
        update_time  
    FROM dim_score_statistics_sink
    WHERE is_current = TRUE
),
-- 筛选出需要更新的 cls（score 变化或新 cls）
changed_cls AS (
    SELECT 
        newa.cls,
        newa.new_score,
        olda.id AS old_id,
        olda.old_score,
        olda.dw_start_time AS old_dw_start_time,
        olda.dw_end_time AS old_dw_end_time,
        olda.update_time AS old_update_time  
    FROM newa_score AS newa
    LEFT JOIN olda_current AS olda ON newa.cls = olda.cls
    WHERE newa.new_score <> olda.old_score OR olda.cls IS NULL
),
-- 生成新记录（仅当需要变化时）
newa_records AS (
    SELECT 
        cls,
        '9999-12-31 23:59:59' AS dw_end_time,
        TRUE AS is_current,
        UUID() AS id,
        new_score AS score,
        DATE_FORMAT(CURRENT_TIMESTAMP, 'yyyy-MM-dd HH:mm:ss') AS update_time,
        DATE_FORMAT(CURRENT_TIMESTAMP, 'yyyy-MM-dd HH:mm:ss') AS dw_start_time
    FROM changed_cls
),
-- 生成失效的旧记录（仅当 score 变化时）
closed_olda_records AS (
    SELECT 
        olda.cls AS cls,
        DATE_FORMAT(CURRENT_TIMESTAMP, 'yyyy-MM-dd HH:mm:ss') AS dw_end_time,
        FALSE AS is_current,
        olda.id AS id,
        olda.old_score AS score,
        olda.update_time AS update_time,  
        olda.dw_start_time AS dw_start_time  -- 修正：直接引用 olda_current 的 dw_start_time
    FROM changed_cls AS changed 
    JOIN olda_current AS olda ON changed.cls = olda.cls 
    WHERE changed.new_score <> olda.old_score
)
-- 合并新记录和失效的旧记录
SELECT 
    cls,
    dw_end_time,
    is_current,
    id,
    score,
    update_time,
    dw_start_time
FROM newa_records
UNION ALL
SELECT 
    cls,
    dw_end_time,
    is_current,
    id,
    score,
    update_time,
    dw_start_time
FROM closed_olda_records;
相关推荐
iVictor1 小时前
分享一个 MySQL binlog 分析小工具
mysql
老华带你飞1 小时前
口腔助手|口腔挂号预约小程序|基于微信小程序的口腔门诊预约系统的设计与实现(源码+数据库+文档)
java·数据库·微信小程序·小程序·论文·毕设·口腔小程序
hqxstudying1 小时前
J2EE模式---服务层模式
java·数据库·后端·spring·oracle·java-ee
Yu_Lijing1 小时前
MySQL进阶学习与初阶复习第四天
数据库·学习·mysql
大熊程序猿1 小时前
net8.0一键创建支持(Orm-Sqlite-MySql-SqlServer)
数据库·mysql·sqlite
-SGlow-10 小时前
MySQL相关概念和易错知识点（2）（表结构的操作、数据类型、约束）
linux·运维·服务器·数据库·mysql
明月56611 小时前
Oracle 误删数据恢复
数据库·oracle
水瓶_bxt12 小时前
Centos安装HAProxy搭建Mysql高可用集群负载均衡
mysql·centos·负载均衡
♡喜欢做梦12 小时前
【MySQL】深入浅出事务：保证数据一致性的核心武器
数据库·mysql
遇见你的雩风13 小时前
MySQL的认识与基本操作
数据库·mysql
热门推荐
01Qwen3-Coder 快速上手教程 | Qwen Code + Claude Code02vue数据变化但页面不变03全球最强模型Grok4，国内已可免费使用！（附教程）04扣子开源本地部署教程 丨Coze智能体小白喂饭级指南05KGG转MP3工具|非KGM文件|解密音频06sqli-labs 靶场 less-8、9、10 第八关到第十关详解：布尔注入，时间注入07干翻 Typora！MilkUp：完全免费的桌面端 Markdown 编辑器！08Coze 开源了，送上保姆级私有化部署方案【建议收藏】09【2025.7.18】更新vscode后所有.vue文件template标签后报红的临时解决办法，Vue - Official 插件3.0.2导致1001-开源版COZE-字节 Coze Studio 重磅开源！保姆级本地安装教程，手把手带你体验