Flink CDC3.X + Flink1.19环境搭建

sl43792024-07-07 23:33

pom.xml

XML 复制代码 
 <properties>
    <maven.compiler.source>8</maven.compiler.source>
    <scala.binary.version>2.12</scala.binary.version>
    <flink.version>1.19.0</flink.version>
    <log4j.version>1.2.17</log4j.version>
    <slf4j.version>1.7.21</slf4j.version>
    <fastjson.version>2.0.20</fastjson.version>
    <avro.version>1.11.0</avro.version>
    <okhttp.version>3.6.0</okhttp.version>
    <hikari.cp.version>2.6.1</hikari.cp.version>
    <avro.version>1.10.0</avro.version>
    <jedis.version>4.2.0</jedis.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

  <dependencies>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-core -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-core</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-java -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-java</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-java -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-java</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-clients</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-json -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-json</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-files -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-files</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-datagen</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-statebackend-rocksdb -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-statebackend-rocksdb</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-planner -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-table-planner_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-kafka</artifactId>
      <version>3.2.0-1.19</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-hive -->
<!--    <dependency>-->
<!--      <groupId>org.apache.flink</groupId>-->
<!--      <artifactId>flink-connector-hive_2.12</artifactId>-->
<!--       <version>${flink.version}</version>-->
<!--      <scope>provided</scope>-->
<!--    </dependency>-->

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-jdbc -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-jdbc</artifactId>
      <version>3.2.0-1.19</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>com.ververica</groupId>
      <artifactId>flink-sql-connector-mysql-cdc</artifactId>
      <version>3.0.1</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/com.mysql/mysql-connector-j -->
    <dependency>
      <groupId>com.mysql</groupId>
      <artifactId>mysql-connector-j</artifactId>
      <version>8.0.33</version>
    </dependency>

    <!--    <dependency>-->
<!--      <groupId>org.apache.hadoop</groupId>-->
<!--      <artifactId>hadoop-client</artifactId>-->
<!--      <version>3.3.5</version>-->
<!--    </dependency>-->

    <!-- https://mvnrepository.com/artifact/redis.clients/jedis -->
    <dependency>
      <groupId>redis.clients</groupId>
      <artifactId>jedis</artifactId>
      <version>5.0.0</version>
    </dependency>

    <!-- 日志打印的jar包 -->
    <dependency>
      <groupId>log4j</groupId>
      <artifactId>log4j</artifactId>
      <version>${log4j.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
      <version>${slf4j.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
      <version>${slf4j.version}</version>
      <scope>provided</scope>
    </dependency>

  </dependencies>
  <build>
    <finalName>flink-demo</finalName>

    <plugins>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>3.1.0</version>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>

              <artifactSet>
                <excludes>
                  <exclude>org.apache.flink:force-shading</exclude>
                  <exclude>com.google.code.findbugs:jsr305</exclude>
                  <exclude>org.slf4j:*</exclude>
                  <exclude>log4j:*</exclude>
                </excludes>
              </artifactSet>

              <transformers>
                <transformer
                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                  <mainClass>com.sl.testFlink</mainClass>
                </transformer>
              </transformers>
              <minimizeJar>false</minimizeJar>
            </configuration>
          </execution>
        </executions>
      </plugin>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <version>3.8.0</version>
        <configuration>
          <source>1.8</source>
          <target>1.8</target>
        </configuration>
      </plugin>
    </plugins>
  </build>

2.代码示例：监控MySQL8：

java 复制代码 
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        Properties dbProps = new Properties();
        //通过FlinkCDC构建SourceFunction
        DebeziumSourceFunction<String> sourceFunction = MySqlSource.<String>builder()
                .hostname("localhost")
                .port(3306)
                .username("root")
                .password("123456")
                .databaseList("project")	//监控的数据库
//                .tableList("cdc_test.user_info")	//监控的数据库下的表
                .deserializer(new JsonDebeziumDeserializationSchema())//反序列化
                .debeziumProperties(dbProps)
                .startupOptions(StartupOptions.latest())
                .build();
        DataStreamSource<String> dataStreamSource = env.addSource(sourceFunction);
        dataStreamSource.print();
        env.execute("testcdc");

3.MySQL配置：

mysqld

#设置3306端口

port=3306

#设置mysql的安装目录，这个大家需要根据自己的安装目录来改一下

#值得一提的是，这个地方建议大家用\\表示一层目录，用\有时候会报错，也不知道为什么

basedir=D:\\install_soft\\mysql-8.0.32-winx64

#设置mysql数据库的数据的存放目录

datadir=D:\\install_soft\\mysql-8.0.32-winx64\\data\\data

#允许最大连接数

max_connections=200

#允许连接失败的次数。

max_connect_errors=10

#服务端使用的字符集默认为utf8mb4

character-set-server=utf8mb4

#创建新表时将使用的默认存储引擎

default-storage-engine=INNODB

innodb_buffer_pool_size=256M

#默认使用"mysql_native_password"插件认证

default_authentication_plugin=mysql_native_password

default-time-zone=+08:00

#主从复制或MGR集群中，server_id记得要不同

server-id=1

#注：该参数代表二进制日志的格式。binlog格式有三种statement、row和mixed。生产环境中使用row这种格式更安全，不会出现跨库复制丢数据的情况

binlog_format="ROW"

log_bin=D:\\install_soft\\mysql-8.0.32-winx64\\mysql_bin

max_binlog_size = 1G

注：该参数代表binlog的过期时间，单位是天

expire_logs_days = 10

注：慢查询日志的开关，该参数等于1代表开启慢查询，查看慢查询写入位置：show variables like 'slow_query_log_file';

#slow_query_log = 0

注：慢查询的时间，某条SQL语句超过该参数设置的时间，就会记录到慢查询日志中。单位是秒

#long_query_time = 3

注：表名是否区分大小的参数。默认是值为0。0代表区分大小写，1代表不区分大小写，以小写存储

lower_case_table_names = 1

相关推荐
武子康2 小时前
大数据-237 离线数仓 - Hive 广告业务实战：ODS→DWD 事件解析、广告明细与转化分析落地
大数据·后端·apache hive
大大大大晴天4 小时前
Flink生产问题排障-Kryo serializer scala extensions are not available
大数据·flink
武子康2 天前
大数据-236 离线数仓 - 会员指标验证、DataX 导出与广告业务 ODS/DWD/ADS 全流程
大数据·后端·apache hive
武子康3 天前
大数据-235 离线数仓 - 实战：Flume+HDFS+Hive 搭建 ODS/DWD/DWS/ADS 会员分析链路
大数据·后端·apache hive
DianSan_ERP4 天前
电商API接口全链路监控：构建坚不可摧的线上运维防线
大数据·运维·网络·人工智能·git·servlet
够快云库4 天前
能源行业非结构化数据治理实战：从数据沼泽到智能资产
大数据·人工智能·机器学习·企业文件安全
AI周红伟4 天前
周红伟：智能体全栈构建实操：OpenClaw部署+Agent Skills+Seedance+RAG从入门到实战
大数据·人工智能·大模型·智能体
B站计算机毕业设计超人4 天前
计算机毕业设计Django+Vue.js高考推荐系统 高考可视化 大数据毕业设计(源码+LW文档+PPT+详细讲解)
大数据·vue.js·hadoop·django·毕业设计·课程设计·推荐算法
计算机程序猿学长4 天前
大数据毕业设计-基于django的音乐网站数据分析管理系统的设计与实现(源码+LW+部署文档+全bao+远程调试+代码讲解等)
大数据·django·课程设计
B站计算机毕业设计超人4 天前
计算机毕业设计Django+Vue.js音乐推荐系统 音乐可视化 大数据毕业设计 (源码+文档+PPT+讲解)
大数据·vue.js·hadoop·python·spark·django·课程设计
热门推荐
01GitHub 镜像站点02【OpenClaw 本地实战 Ep.3】突破瓶颈：强制修改 openclaw.json 解锁 32k 上下文记忆03OpenClaw 使用和管理 MCP 完全指南04OpenClaw + 飞书（Feishu）环境搭建指南05OpenClaw优化飞书API 额度已耗尽问题06Claude Code + GLM4.7 避坑指南：解决 Unable to connect to Anthropic services07OpenClaw大龙虾机器人完整安装教程08Clawdbot部署教程：解决‘gateway token missing’授权问题的完整步骤09Window 10部署openclaw报错node.exe : npm error code 12810OpenClaw 接入阿里云百炼 Coding Plan 指南