Flink CDC3.X + Flink1.19环境搭建

sl43792024-07-07 23:33

pom.xml

XML 复制代码 
 <properties>
    <maven.compiler.source>8</maven.compiler.source>
    <scala.binary.version>2.12</scala.binary.version>
    <flink.version>1.19.0</flink.version>
    <log4j.version>1.2.17</log4j.version>
    <slf4j.version>1.7.21</slf4j.version>
    <fastjson.version>2.0.20</fastjson.version>
    <avro.version>1.11.0</avro.version>
    <okhttp.version>3.6.0</okhttp.version>
    <hikari.cp.version>2.6.1</hikari.cp.version>
    <avro.version>1.10.0</avro.version>
    <jedis.version>4.2.0</jedis.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

  <dependencies>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-core -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-core</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-java -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-java</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-java -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-java</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-clients</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-json -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-json</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-files -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-files</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-datagen</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-statebackend-rocksdb -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-statebackend-rocksdb</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-planner -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-table-planner_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-kafka</artifactId>
      <version>3.2.0-1.19</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-hive -->
<!--    <dependency>-->
<!--      <groupId>org.apache.flink</groupId>-->
<!--      <artifactId>flink-connector-hive_2.12</artifactId>-->
<!--       <version>${flink.version}</version>-->
<!--      <scope>provided</scope>-->
<!--    </dependency>-->

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-jdbc -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-jdbc</artifactId>
      <version>3.2.0-1.19</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>com.ververica</groupId>
      <artifactId>flink-sql-connector-mysql-cdc</artifactId>
      <version>3.0.1</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/com.mysql/mysql-connector-j -->
    <dependency>
      <groupId>com.mysql</groupId>
      <artifactId>mysql-connector-j</artifactId>
      <version>8.0.33</version>
    </dependency>

    <!--    <dependency>-->
<!--      <groupId>org.apache.hadoop</groupId>-->
<!--      <artifactId>hadoop-client</artifactId>-->
<!--      <version>3.3.5</version>-->
<!--    </dependency>-->

    <!-- https://mvnrepository.com/artifact/redis.clients/jedis -->
    <dependency>
      <groupId>redis.clients</groupId>
      <artifactId>jedis</artifactId>
      <version>5.0.0</version>
    </dependency>

    <!-- 日志打印的jar包 -->
    <dependency>
      <groupId>log4j</groupId>
      <artifactId>log4j</artifactId>
      <version>${log4j.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
      <version>${slf4j.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
      <version>${slf4j.version}</version>
      <scope>provided</scope>
    </dependency>

  </dependencies>
  <build>
    <finalName>flink-demo</finalName>

    <plugins>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>3.1.0</version>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>

              <artifactSet>
                <excludes>
                  <exclude>org.apache.flink:force-shading</exclude>
                  <exclude>com.google.code.findbugs:jsr305</exclude>
                  <exclude>org.slf4j:*</exclude>
                  <exclude>log4j:*</exclude>
                </excludes>
              </artifactSet>

              <transformers>
                <transformer
                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                  <mainClass>com.sl.testFlink</mainClass>
                </transformer>
              </transformers>
              <minimizeJar>false</minimizeJar>
            </configuration>
          </execution>
        </executions>
      </plugin>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <version>3.8.0</version>
        <configuration>
          <source>1.8</source>
          <target>1.8</target>
        </configuration>
      </plugin>
    </plugins>
  </build>

2.代码示例：监控MySQL8：

java 复制代码 
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        Properties dbProps = new Properties();
        //通过FlinkCDC构建SourceFunction
        DebeziumSourceFunction<String> sourceFunction = MySqlSource.<String>builder()
                .hostname("localhost")
                .port(3306)
                .username("root")
                .password("123456")
                .databaseList("project")	//监控的数据库
//                .tableList("cdc_test.user_info")	//监控的数据库下的表
                .deserializer(new JsonDebeziumDeserializationSchema())//反序列化
                .debeziumProperties(dbProps)
                .startupOptions(StartupOptions.latest())
                .build();
        DataStreamSource<String> dataStreamSource = env.addSource(sourceFunction);
        dataStreamSource.print();
        env.execute("testcdc");

3.MySQL配置：

[mysqld]

#设置3306端口

port=3306

#设置mysql的安装目录，这个大家需要根据自己的安装目录来改一下

#值得一提的是，这个地方建议大家用\\表示一层目录，用\有时候会报错，也不知道为什么

basedir=D:\\install_soft\\mysql-8.0.32-winx64

#设置mysql数据库的数据的存放目录

datadir=D:\\install_soft\\mysql-8.0.32-winx64\\data\\data

#允许最大连接数

max_connections=200

#允许连接失败的次数。

max_connect_errors=10

#服务端使用的字符集默认为utf8mb4

character-set-server=utf8mb4

#创建新表时将使用的默认存储引擎

default-storage-engine=INNODB

innodb_buffer_pool_size=256M

#默认使用"mysql_native_password"插件认证

default_authentication_plugin=mysql_native_password

default-time-zone=+08:00

#主从复制或MGR集群中，server_id记得要不同

server-id=1

#注：该参数代表二进制日志的格式。binlog格式有三种statement、row和mixed。生产环境中使用row这种格式更安全，不会出现跨库复制丢数据的情况

binlog_format="ROW"

log_bin=D:\\install_soft\\mysql-8.0.32-winx64\\mysql_bin

max_binlog_size = 1G

注：该参数代表binlog的过期时间，单位是天

expire_logs_days = 10

注：慢查询日志的开关，该参数等于1代表开启慢查询，查看慢查询写入位置：show variables like 'slow_query_log_file';

#slow_query_log = 0

注：慢查询的时间，某条SQL语句超过该参数设置的时间，就会记录到慢查询日志中。单位是秒

#long_query_time = 3

注：表名是否区分大小的参数。默认是值为0。0代表区分大小写，1代表不区分大小写，以小写存储

lower_case_table_names = 1

相关推荐
szxinmai主板定制专家2 小时前
【国产NI替代】基于FPGA的32通道（24bits）高精度终端采集核心板卡
大数据·人工智能·fpga开发
TGB-Earnest3 小时前
【py脚本+logstash+es实现自动化检测工具】
大数据·elasticsearch·自动化
大圣数据星球5 小时前
Fluss 写入数据湖实战
大数据·设计模式·flink
suweijie7685 小时前
SpringCloudAlibaba | Sentinel从基础到进阶
java·大数据·sentinel
Data跳动11 小时前
Spark内存都消耗在哪里了？
大数据·分布式·spark
woshiabc11112 小时前
windows安装Elasticsearch及增删改查操作
大数据·elasticsearch·搜索引擎
lucky_syq12 小时前
Saprk和Flink的区别
大数据·flink
lucky_syq12 小时前
流式处理，为什么Flink比Spark Streaming好？
大数据·flink·spark
袋鼠云数栈12 小时前
深入浅出Flink CEP丨如何通过Flink SQL作业动态更新Flink CEP作业
大数据
热门推荐
01〔AI 绘画〕Stable Diffusion 之 解决绘制多人或面部很小的人物时面部崩坏问题 篇02ARM学习（31）编译器对overlay方式的支持03玄机平台应急响应—webshell查杀04DisplayPort AltMode On Type-c协议_Ver2.0(学习笔记)05校验 GPT-4 真实性的三个经典问题：快速区分 GPT-3.5 与 GPT-4，并提供免费测试网站06Docker 夺命连环 15 问076个主流的工业3D管道设计软件08基于大语言模型（LLM）的合成数据生成、策展和评估的综述09(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明10红米手机使用google play