Flink CDC3.X + Flink1.19环境搭建

sl43792024-07-07 23:33

pom.xml

XML 复制代码 
 <properties>
    <maven.compiler.source>8</maven.compiler.source>
    <scala.binary.version>2.12</scala.binary.version>
    <flink.version>1.19.0</flink.version>
    <log4j.version>1.2.17</log4j.version>
    <slf4j.version>1.7.21</slf4j.version>
    <fastjson.version>2.0.20</fastjson.version>
    <avro.version>1.11.0</avro.version>
    <okhttp.version>3.6.0</okhttp.version>
    <hikari.cp.version>2.6.1</hikari.cp.version>
    <avro.version>1.10.0</avro.version>
    <jedis.version>4.2.0</jedis.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
  </properties>

  <dependencies>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-core -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-core</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-java -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-java</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-java -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-java</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-clients</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-json -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-json</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-files -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-files</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-datagen</artifactId>
      <version>${flink.version}</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-statebackend-rocksdb -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-statebackend-rocksdb</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-planner -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-table-planner_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-kafka</artifactId>
      <version>3.2.0-1.19</version>
    </dependency>

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-hive -->
<!--    <dependency>-->
<!--      <groupId>org.apache.flink</groupId>-->
<!--      <artifactId>flink-connector-hive_2.12</artifactId>-->
<!--       <version>${flink.version}</version>-->
<!--      <scope>provided</scope>-->
<!--    </dependency>-->

    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-jdbc -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-jdbc</artifactId>
      <version>3.2.0-1.19</version>
      <scope>provided</scope>
    </dependency>

    <dependency>
      <groupId>com.ververica</groupId>
      <artifactId>flink-sql-connector-mysql-cdc</artifactId>
      <version>3.0.1</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/com.mysql/mysql-connector-j -->
    <dependency>
      <groupId>com.mysql</groupId>
      <artifactId>mysql-connector-j</artifactId>
      <version>8.0.33</version>
    </dependency>

    <!--    <dependency>-->
<!--      <groupId>org.apache.hadoop</groupId>-->
<!--      <artifactId>hadoop-client</artifactId>-->
<!--      <version>3.3.5</version>-->
<!--    </dependency>-->

    <!-- https://mvnrepository.com/artifact/redis.clients/jedis -->
    <dependency>
      <groupId>redis.clients</groupId>
      <artifactId>jedis</artifactId>
      <version>5.0.0</version>
    </dependency>

    <!-- 日志打印的jar包 -->
    <dependency>
      <groupId>log4j</groupId>
      <artifactId>log4j</artifactId>
      <version>${log4j.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-api</artifactId>
      <version>${slf4j.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.slf4j</groupId>
      <artifactId>slf4j-log4j12</artifactId>
      <version>${slf4j.version}</version>
      <scope>provided</scope>
    </dependency>

  </dependencies>
  <build>
    <finalName>flink-demo</finalName>

    <plugins>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <version>3.1.0</version>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>

              <artifactSet>
                <excludes>
                  <exclude>org.apache.flink:force-shading</exclude>
                  <exclude>com.google.code.findbugs:jsr305</exclude>
                  <exclude>org.slf4j:*</exclude>
                  <exclude>log4j:*</exclude>
                </excludes>
              </artifactSet>

              <transformers>
                <transformer
                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                  <mainClass>com.sl.testFlink</mainClass>
                </transformer>
              </transformers>
              <minimizeJar>false</minimizeJar>
            </configuration>
          </execution>
        </executions>
      </plugin>

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <version>3.8.0</version>
        <configuration>
          <source>1.8</source>
          <target>1.8</target>
        </configuration>
      </plugin>
    </plugins>
  </build>

2.代码示例：监控MySQL8：

java 复制代码 
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        Properties dbProps = new Properties();
        //通过FlinkCDC构建SourceFunction
        DebeziumSourceFunction<String> sourceFunction = MySqlSource.<String>builder()
                .hostname("localhost")
                .port(3306)
                .username("root")
                .password("123456")
                .databaseList("project")	//监控的数据库
//                .tableList("cdc_test.user_info")	//监控的数据库下的表
                .deserializer(new JsonDebeziumDeserializationSchema())//反序列化
                .debeziumProperties(dbProps)
                .startupOptions(StartupOptions.latest())
                .build();
        DataStreamSource<String> dataStreamSource = env.addSource(sourceFunction);
        dataStreamSource.print();
        env.execute("testcdc");

3.MySQL配置：

[mysqld]

#设置3306端口

port=3306

#设置mysql的安装目录，这个大家需要根据自己的安装目录来改一下

#值得一提的是，这个地方建议大家用\\表示一层目录，用\有时候会报错，也不知道为什么

basedir=D:\\install_soft\\mysql-8.0.32-winx64

#设置mysql数据库的数据的存放目录

datadir=D:\\install_soft\\mysql-8.0.32-winx64\\data\\data

#允许最大连接数

max_connections=200

#允许连接失败的次数。

max_connect_errors=10

#服务端使用的字符集默认为utf8mb4

character-set-server=utf8mb4

#创建新表时将使用的默认存储引擎

default-storage-engine=INNODB

innodb_buffer_pool_size=256M

#默认使用"mysql_native_password"插件认证

default_authentication_plugin=mysql_native_password

default-time-zone=+08:00

#主从复制或MGR集群中，server_id记得要不同

server-id=1

#注：该参数代表二进制日志的格式。binlog格式有三种statement、row和mixed。生产环境中使用row这种格式更安全，不会出现跨库复制丢数据的情况

binlog_format="ROW"

log_bin=D:\\install_soft\\mysql-8.0.32-winx64\\mysql_bin

max_binlog_size = 1G

注：该参数代表binlog的过期时间，单位是天

expire_logs_days = 10

注：慢查询日志的开关，该参数等于1代表开启慢查询，查看慢查询写入位置：show variables like 'slow_query_log_file';

#slow_query_log = 0

注：慢查询的时间，某条SQL语句超过该参数设置的时间，就会记录到慢查询日志中。单位是秒

#long_query_time = 3

注：表名是否区分大小的参数。默认是值为0。0代表区分大小写，1代表不区分大小写，以小写存储

lower_case_table_names = 1

相关推荐
zhixingheyi_tian5 小时前
Spark 之 Aggregate
大数据·分布式·spark
PersistJiao5 小时前
Spark 分布式计算中网络传输和序列化的关系（一）
大数据·网络·spark
宅小海8 小时前
scala String
大数据·开发语言·scala
小白的白是白痴的白8 小时前
11.17 Scala练习：梦想清单管理
大数据
java1234_小锋8 小时前
Elasticsearch是如何实现Master选举的？
大数据·elasticsearch·搜索引擎
宝哥大数据9 小时前
Flink Joins
flink
Java 第一深情12 小时前
零基础入门Flink，掌握基本使用方法
大数据·flink·实时计算
我的K840912 小时前
Flink整合Hudi及使用
linux·服务器·flink
MXsoft61812 小时前
华为服务器（iBMC）硬件监控指标解读
大数据·运维·数据库
PersistJiao13 小时前
Spark 分布式计算中网络传输和序列化的关系（二）
大数据·网络·spark·序列化·分布式计算
热门推荐
01(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明02PyTorch机器学习实现液态神经网络03【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总04玄机平台应急响应—webshell查杀05Coze扣子平台完整体验和实践（附国内和国际版对比）06Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO07【目标跟踪】相机运动补偿08Unity中PICO实现 隔空取物 和 接触抓取物体09RAG 实践- Ollama+RagFlow 部署本地知识库10怎样让音频速度变慢？请跟随以下方法进行操作