Flink将Hive数据导入StarRocks中任务重跑如何避免数据重复？

MARSERERER2026-10-31 10:39

Hive表中其实没有主键的概念，但是设计StarRocks表时可以增加concat_id这个逻辑主键。我们可以将所有的字段都添加到主键中，变成一个联合主键：

复制代码

CREATE TABLE `bikeda_ebik_saas_user_ride_assort_gather_df_v1` (
  `concat_id` varchar(65533) NOT NULL COMMENT "",
  `pt` date NOT NULL COMMENT "",
  `tenant_id` varchar(65533) NOT NULL COMMENT "",
  `city_guid` bigint NOT NULL COMMENT "",
) ENGINE=OLAP 
PRIMARY KEY(`concat_id`, `pt`)
COMMENT "测试表"
PARTITION BY date_trunc('day', pt)
DISTRIBUTED BY HASH(`concat_id`)
PROPERTIES (
"replication_num" = "3",
"in_memory" = "false",
"enable_persistent_index" = "true",
"replicated_storage" = "true",
"partition_live_number" = "62",
"compression" = "LZ4"
);

Flink写入时用concat_ws把所有字段连接在一起：

复制代码

-- 创建Sink，链接StarRocks的表
CREATE TABLE IF NOT EXISTS `df_v1` (
  `concat_id` varchar(65533) NOT NULL,
  `pt` date not null,
  `tenant_id` string not null,
  `city_guid` bigint not null,
  PRIMARY KEY (concat_id, pt) NOT ENFORCED
) WITH (
  'connector' = 'starrocks',
  'jdbc-url' = 'jdbc:mysql:loadbalance://:9030,:9030,:9030',
  'load-url' = ':8030;:8030;:8030',
  'database-name' = 'test',
  'table-name' = 'df',
  'username' = 'root',
  'password' = '123456'
);
-- 创建hive catalog
-- pro生产环境中hive-conf-dir为/app/hive/conf
-- fat测试环境中hive-conf-dir为/app/hive/dev-conf
CREATE CATALOG flinkHiveCatalog WITH (
  'type' = 'hive',
  'default-database' = 'default',
  'hive-conf-dir' = '/app/hive/conf'
);

-- 使用hive catalog
-- 直接使用hive catalog，只是为了的select部分查询hive表，写法方便
-- 查询hive部分直接db.table即可，不用catalog.db.table三段名字
USE CATALOG flinkHiveCatalog;

-- insert语句插入StarRocks
-- 注意，这时候默认catalog是flinkHiveCatalog，所以StarRocks的sink表要加default_catalog和default_database
insert into
  default_catalog.default_database.df_v1
select
    concat_ws('\001',CAST(TO_TIMESTAMP(CAST(`pt` AS STRING), 'yyyyMMdd') AS VARCHAR),coalesce(`tenant_id`, ''),CAST(coalesce(`city_guid`, -777) AS VARCHAR),),
    TO_TIMESTAMP(CAST(`pt` AS STRING), 'yyyyMMdd'),
    coalesce(`tenant_id`, ''),
    coalesce(`city_guid`, '')
    
from test.df
where pt = '${dt}'