第 4 篇：「Fluss + Flink 集成实战」—— Catalog、Source 与 Sink

第 4 篇：「Fluss + Flink 集成实战」------ Catalog、Source 与 Sink

阅读本文你将了解： 如何使用 Fluss 作为 Flink 的原生 Catalog、如何读写 Fluss 表、DataStream API 集成、以及一个完整的实时订单宽表构建案例。

4.1 Fluss Catalog 注册与配置

4.1.1 Maven 依赖

xml 复制代码

<properties>
    <flink.version>1.20.3</flink.version>
    <fluss.version>0.9.1</fluss.version>
</properties>

<dependencies>
    <!-- Flink 核心 -->
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-table-api-java</artifactId>
        <version>${flink.version}</version>
    </dependency>
    
    <!-- Fluss Flink Connector -->
    <dependency>
        <groupId>org.apache.fluss</groupId>
        <artifactId>fluss-flink-1.20</artifactId>
        <version>${fluss.version}</version>
    </dependency>
    
    <!-- Flink SQL Client 运行时依赖 -->
    <dependency>
        <groupId>org.apache.flink</groupId>
        <artifactId>flink-table-planner_2.12</artifactId>
        <version>${flink.version}</version>
    </dependency>
</dependencies>

4.1.2 Catalog 注册

sql 复制代码

-- Flink SQL Client 中执行
CREATE CATALOG fluss_catalog WITH (
    'type' = 'fluss',
    'bootstrap.servers' = 'coordinator-server-1:9123,coordinator-server-2:9123'
);

USE CATALOG fluss_catalog;

-- 验证 Catalog 已注册
SHOW CATALOGS;
-- 输出：fluss_catalog (当前), default_catalog

4.1.3 Java API 方式注册

java 复制代码

import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;

public class FlussFlinkIntegration {
    
    public static void main(String[] args) {
        EnvironmentSettings settings = EnvironmentSettings
            .newInstance()
            .inStreamingMode()
            .build();
        
        TableEnvironment tEnv = TableEnvironment.create(settings);
        
        // 注册 Fluss Catalog
        tEnv.executeSql(
            "CREATE CATALOG fluss_catalog WITH (\n" +
            "    'type' = 'fluss',\n" +
            "    'bootstrap.servers' = 'localhost:9123'\n" +
            ")"
        );
        
        tEnv.executeSql("USE CATALOG fluss_catalog");
        
        System.out.println("Fluss Catalog registered successfully");
    }
}

4.1.4 FlussCatalog 源码分析

java 复制代码

// 简化自 org.apache.fluss.flink.catalog.FlussCatalog
public class FlussCatalog extends AbstractCatalog {
    
    private final FlussConnection connection;
    private final String bootstrapServers;
    
    @Override
    public void open() throws CatalogException {
        // 建立与 Fluss CoordinatorServer 的连接
        this.connection = FlussConnection.create(
            FlussConfig.builder()
                .setBootstrapServers(bootstrapServers)
                .build()
        );
    }
    
    @Override
    public CatalogBaseTable getTable(ObjectPath tablePath) {
        // 从 CoordinatorServer 获取表元数据
        TableDescriptor descriptor = connection.getTable(tablePath);
        
        // 将 Fluss Schema 转换为 Flink Schema
        Schema flinkSchema = SchemaConverter.toFlinkSchema(descriptor.getSchema());
        
        // 根据表类型创建对应的 Flink Connector Table
        if (descriptor.getTableType() == TableType.PRIMARY_KEY) {
            return CatalogTable.of(
                flinkSchema,
                "PK table backed by Fluss",
                Collections.emptyList(),
                getTableOptions(descriptor)
            );
        } else {
            return CatalogTable.of(
                flinkSchema,
                "Log table backed by Fluss", 
                Collections.emptyList(),
                getTableOptions(descriptor)
            );
        }
    }
}

4.2 创建和管理表

sql 复制代码

USE CATALOG fluss_catalog;
CREATE DATABASE ecommerce;
USE ecommerce;

-- 创建数据源表（Source）
CREATE TABLE order_source (
    order_id    BIGINT,
    user_id     BIGINT,
    product_id  BIGINT,
    amount      DECIMAL(10, 2),
    order_time  TIMESTAMP(3),
    PRIMARY KEY (order_id) NOT ENFORCED
) WITH (
    'bucket.num' = '8',
    'table.merge-engine' = 'deduplicate'
);

-- 创建维表
CREATE TABLE product_dim (
    product_id   BIGINT,
    product_name STRING,
    category     STRING,
    price        DECIMAL(10, 2),
    PRIMARY KEY (product_id) NOT ENFORCED
) WITH (
    'bucket.num' = '4'
);

-- 创建结果表（Sink）
CREATE TABLE order_wide (
    order_id      BIGINT,
    user_id       BIGINT,
    amount        DECIMAL(10, 2),
    product_name  STRING,
    category      STRING,
    order_time    TIMESTAMP(3),
    PRIMARY KEY (order_id) NOT ENFORCED
) WITH (
    'bucket.num' = '16'
);

4.3 实时数据写入

4.3.1 Flink SQL 写入

sql 复制代码

-- 单表写入
INSERT INTO product_dim VALUES
    (1, 'iPhone 15', 'Electronics', 6999.00),
    (2, 'Nike Air Max', 'Shoes', 899.00),
    (3, 'Coffee Maker', 'Home', 299.00);

-- 多表并行写入（使用 STATEMENT SET）
EXECUTE STATEMENT SET
BEGIN
    INSERT INTO product_dim VALUES (4, 'MacBook Pro', 'Electronics', 12999.00);
    INSERT INTO product_dim VALUES (5, 'Levi\'s Jeans', 'Clothing', 499.00);
END;

4.3.2 DataStream API 写入

java 复制代码

import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;

public class FlinkDataStreamWrite {
    
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
        
        // 注册 Catalog
        tEnv.executeSql("CREATE CATALOG fluss_catalog WITH ('type'='fluss','bootstrap.servers'='localhost:9123')");
        tEnv.executeSql("USE CATALOG fluss_catalog");
        
        // 创建模拟数据源
        DataStream<Order> orderStream = env
            .addSource(new OrderGenerator())  // 自定义 Source
            .name("Order Generator");
        
        // 注册为临时表
        tEnv.createTemporaryView("temp_orders", orderStream);
        
        // 写入 Fluss 表
        tEnv.executeSql(
            "INSERT INTO ecommerce.order_source " +
            "SELECT order_id, user_id, product_id, amount, order_time " +
            "FROM temp_orders"
        ).await();
    }
}

4.3.3 写入语义保证

Fluss Flink Sink 提供 Exactly-Once 保证：

java 复制代码

// Fluss Flink Sink 的 Exactly-Once 实现
// 简化自 Sink 内部逻辑

// Checkpoint 时：
// 1. Flink 触发 Checkpoint
// 2. Fluss Sink flush 所有 pending 的数据
// 3. Sink 将当前 offset 写入 Checkpoint
// 4. 数据在所有 ISR 副本确认持久化后，Checkpoint 完成

// 故障恢复时：
// 1. Flink 从最近完成的 Checkpoint 恢复
// 2. Sink 从 Checkpoint 记录的 offset 继续写入
// 3. 不会产生重复数据

4.4 流式与批量读取

4.4.1 流式读取（Streaming Mode）

sql 复制代码

-- 设置为流式执行模式
SET 'execution.runtime-mode' = 'streaming';

-- 实时消费订单数据
SELECT 
    order_id, user_id, amount,
    TUMBLE_START(order_time, INTERVAL '1' MINUTE) AS window_start,
    SUM(amount) AS total_amount,
    COUNT(*) AS order_count
FROM order_source
GROUP BY 
    order_id, user_id, amount,
    TUMBLE(order_time, INTERVAL '1' MINUTE);

4.4.2 批量读取（Batch Mode）

sql 复制代码

-- 设置为批量执行模式
SET 'execution.runtime-mode' = 'batch';

-- 历史数据分析
SELECT 
    DATE_FORMAT(order_time, 'yyyy-MM-dd') AS order_date,
    COUNT(*) AS total_orders,
    SUM(amount) AS revenue,
    AVG(amount) AS avg_order_value
FROM order_source
WHERE order_time >= TIMESTAMP '2026-08-01 00:00:00'
GROUP BY DATE_FORMAT(order_time, 'yyyy-MM-dd')
ORDER BY order_date;

4.4.3 流批模式对比

java 复制代码

// Java API 控制执行模式
public class StreamBatchSwitch {
    
    public static void main(String[] args) {
        // 流式模式
        EnvironmentSettings streamingSettings = EnvironmentSettings
            .newInstance()
            .inStreamingMode()  // ← 关键配置
            .build();
        
        // 批量模式
        EnvironmentSettings batchSettings = EnvironmentSettings
            .newInstance()
            .inBatchMode()      // ← 关键配置
            .build();
        
        // 流批读取同一个 Fluss 表，无需修改 SQL
        // Fluss 会根据模式自动优化执行计划
    }
}

4.5 更新与删除操作

sql 复制代码

-- 更新操作（PK 表）
UPDATE product_dim 
SET price = 7599.00 
WHERE product_id = 1;

-- 删除操作（PK 表）
DELETE FROM product_dim WHERE product_id = 99;

-- 批量更新
UPDATE order_source 
SET amount = amount * 1.1 
WHERE order_time >= TIMESTAMP '2026-08-01 00:00:00';

更新/删除的源码路径

复制代码

org.apache.fluss.flink.sink
├── FlussSink.java              # Flink Sink 入口
├── FlussWriter.java            # 写入逻辑
└── writer/
    ├── UpsertWriter.java       # Upsert（Insert/Update）写入
    ├── AppendWriter.java       # 追加写入（Log Table）
    └── DeleteWriter.java       # 删除写入

4.6 实战案例：实时订单宽表构建

4.6.1 架构

复制代码

┌─────────────┐     ┌─────────────┐
│ order_source │     │ product_dim  │
│  (订单流水)   │     │  (商品维表)   │
└──────┬───────┘     └──────┬──────┘
       │                    │
       └────────┬───────────┘
                │
       ┌────────▼────────┐
       │   Flink Job      │
       │  Lookup Join     │
       └────────┬────────┘
                │
       ┌────────▼────────┐
       │   order_wide     │
       │  (订单宽表)       │
       └─────────────────┘

4.6.2 完整实现

sql 复制代码

-- Step 1: 创建 Catalog 和 Database
CREATE CATALOG fluss_catalog WITH (
    'type' = 'fluss',
    'bootstrap.servers' = 'localhost:9123'
);
USE CATALOG fluss_catalog;
CREATE DATABASE ecommerce;
USE ecommerce;

-- Step 2: 创建表（使用前面定义的表）

-- Step 3: 初始化维表数据
INSERT INTO product_dim VALUES
    (1, 'iPhone 15', 'Electronics', 6999.00),
    (2, 'Nike Air Max', 'Shoes', 899.00);

-- Step 4: 实时宽表构建
SET 'execution.runtime-mode' = 'streaming';

INSERT INTO order_wide
SELECT 
    o.order_id,
    o.user_id,
    o.amount,
    p.product_name,
    p.category,
    o.order_time
FROM order_source AS o
LEFT JOIN product_dim FOR SYSTEM_TIME AS OF o.order_time AS p
    ON o.product_id = p.product_id;

-- Step 5: 验证宽表数据
SELECT * FROM order_wide;

4.6.3 Java 代码版本

java 复制代码

public class OrderWideTableJob {
    
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 每 60 秒一次 Checkpoint
        env.enableCheckpointing(60000);
        
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
        
        // 注册 Catalog
        tEnv.executeSql(
            "CREATE CATALOG fluss_catalog WITH (" +
            "  'type'='fluss'," +
            "  'bootstrap.servers'='localhost:9123'" +
            ")"
        );
        tEnv.executeSql("USE CATALOG fluss_catalog");
        tEnv.executeSql("USE ecommerce");
        
        // 实时宽表构建
        tEnv.executeSql(
            "INSERT INTO order_wide " +
            "SELECT o.order_id, o.user_id, o.amount, " +
            "       p.product_name, p.category, o.order_time " +
            "FROM order_source AS o " +
            "LEFT JOIN product_dim FOR SYSTEM_TIME AS OF o.order_time AS p " +
            "  ON o.product_id = p.product_id"
        ).await();
    }
}

4.7 总结与下一篇预告

操作	Flink SQL 示例	执行模式
注册 Catalog	`CREATE CATALOG ... WITH ('type'='fluss')`	一次注册，全局可用
创建表	`CREATE TABLE ... PRIMARY KEY ... NOT ENFORCED`	DDL
流式写入	`INSERT INTO ... VALUES ...`	Streaming
流式读取	`SELECT ... FROM ...` + `SET streaming`	Streaming
批量读取	`SELECT ... FROM ...` + `SET batch`	Batch
更新删除	`UPDATE/DELETE ... WHERE ...`	DML
Lookup Join	`LEFT JOIN ... FOR SYSTEM_TIME AS OF ...`	Streaming

下一篇我们将深入 Fluss 的列式流处理能力------Apache Arrow 格式如何工作、列裁剪和谓词下推如何将 I/O 降低数量级。

本文基于 Apache Fluss 0.9.1 + Apache Flink 1.20.3。源码: https://github.com/apache/fluss