使用finksql方式将mysql数据同步到kafka中,每次只能同步一张表

使用finksql方式将mysql数据同步到kafka中,每次只能同步一张表

bash 复制代码
package flink;


import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;


public class FlinkSQL_CDC {

    public static void main(String[] args) throws Exception {

//
//        Configuration conf = new Configuration();
//        conf.setInteger("rest.port",3335);
//        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(conf);

        //1.创建执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);


        //2.创建Flink-MySQL-CDC的Source
        TableResult tableResult = tableEnv.executeSql("CREATE TABLE table_name (" +
                "  id INT primary key," +
                "  name STRING" +
                ") WITH (" +
                "  'connector' = 'mysql-cdc'," +
                "  'hostname' = 'hadoop102'," +
                "  'port' = '3306'," +
                "  'username' = 'root'," +
                "  'password' = 'xxxx'," +
                "  'database-name' = 'student'," +
                "  'table-name' = 'table_name'," +
                "'server-time-zone' = 'Asia/Shanghai'," +
                "'scan.startup.mode' = 'initial'" +
                ")"
        );

        // 2. 注册SinkTable: sink_sensor
//        tableEnv.executeSql("" +
//                "CREATE TABLE kafka_binlog ( " +
//                "  user_id INT, " +
//                "  user_name STRING, " +
//                "`proc_time` as PROCTIME()" +
//                ") WITH ( " +
//                "  'connector' = 'kafka', " +
//                "  'topic' = 'test2', " +
//                "  'properties.bootstrap.servers' = 'hadoop102:9092', " +
//                "  'format' = 'json' " +
//                ")" +
//                "");

        //upsert-kafka
        tableEnv.executeSql("" +
                "CREATE TABLE kafka_binlog ( " +
                "  user_id INT, " +
                "  user_name STRING, " +
                "`proc_time` as PROCTIME()," +
                "  PRIMARY KEY (user_id) NOT ENFORCED" +
                ") WITH ( " +
                "  'connector' = 'upsert-kafka', " +
                "  'topic' = 'test2', " +
                "  'properties.bootstrap.servers' = 'hadoop102:9092', " +
                "  'key.format' = 'json' ," +
                "  'value.format' = 'json' " +
                ")" +
                "");


        // 3. 从SourceTable 查询数据, 并写入到 SinkTable
         tableEnv.executeSql("insert into kafka_binlog select * from table_name");

         tableEnv.executeSql("select * from kafka_binlog").print();

        env.execute();
    }

}
相关推荐
小徐不徐说1 小时前
C++ 接口与插件设计:从工厂注册到 Qt 动态插件
数据库·c++·qt·系统架构·程序设计
论文复现现场2 小时前
WRF、FEA 能用 L40S 加速吗?求解器检查、断点恢复与长期运行
java·数据库·算法·wrf·有限元分析·gpu计算·断点续算
Decay__3 小时前
04_MySQL DBA 日常运维手册 —— 会话管理、锁与阻塞、事务监控、慢查询与日志巡检
运维·数据库·sql·mysql·架构
心梦缘-雪雁4 小时前
SQL Server 查询数据存在哪个表里
数据库
lusklusklusk5 小时前
Python 巡检脚本:进程运行状态、数据库写入时效检查及邮件通知与告警
数据库·python·oracle
GeW5 小时前
产业新优势基础软件的人才底盘,从Linux与数据库开始
linux·数据库
SelectDB6 小时前
Doris Arrow Flight SQL 高速读取协议实战:与 Snowflake 客户端取数链路对照
大数据·数据库·数据分析
曼岛_6 小时前
[图解密码学]第4章 古典密码
java·数据库·密码学
瀚高PG实验室7 小时前
HGDB V9.0.5 .pgpass 免密登录提示 IsEncryptNon-base64 encoding 警告处理
数据库·postgresql·瀚高数据库
龙腾AI白云7 小时前
大模型检索增强生成(RAG)的技术原理与架构设计
数据库·python·机器学习·知识图谱