一、背景
xxl-job跑批次数据出现无法续跑补偿等机制
二、实现
代码大多数都是伪代码，主要记录下思路方便后面cv，减少跟ai沟通聊东西
1、批次表--sql的话你让ai自己生成一下
sql
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.experimental.Accessors;
import java.time.LocalDateTime;
/**
* <p>
* 客户数据同步进度日志
* </p>
*/
@Data
@Accessors(chain = true)
@TableName("CUSTOMER_SYNC_LOG")
public class CustomerSyncLog {
/**
* 主键
*/
@TableId(value = "ID", type = IdType.AUTO)
private Long id;
/**
* 同步批次号（UUID）
*/
@TableField("SYNC_BATCH_ID")
private String syncBatchId;
/**
* 已同步到的最大 customer_no
*/
@TableField("LAST_CUSTOMER_NO")
private String lastCustomerNo;
/**
* 同步状态：0=进行中, 1=完成, 2=失败
*/
@TableField("SYNC_STATUS")
private Integer syncStatus;
/**
* 已同步条数
*/
@TableField("TOTAL_SYNCED")
private Integer totalSynced;
/**
* 补偿重试次数
*/
@TableField("RETRY_COUNT")
private Integer retryCount;
/**
* 触发来源：1=定时全量 2=补偿续跑
*/
@TableField("TRIGGER_SOURCE")
private Integer triggerSource;
/**
* 最后有进展时间（心跳）
*/
@TableField("HEARTBEAT_TIME")
private LocalDateTime heartbeatTime;
/**
* 开始时间
*/
@TableField("START_TIME")
private LocalDateTime startTime;
/**
* 结束时间
*/
@TableField("END_TIME")
private LocalDateTime endTime;
/**
* 错误信息
*/
@TableField("ERROR_MSG")
private String errorMsg;
/**
* 创建时间
*/
@TableField("CREATE_TIME")
private LocalDateTime createTime;
/**
* 更新时间
*/
@TableField("UPDATE_TIME")
private LocalDateTime updateTime;
}2、定时任务入口
java
import com.xxl.job.core.handler.annotation.XxlJob;
import com.ybt.center.platform.service.system.CustomerSyncEngine;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
@Slf4j
@Component
public class CustomerJob {
@Resource
private CustomerSyncEngine customerSyncEngine;
/**
* 定时同步行内客户信息---同步starRocks客户表信息（keyset分页 + 断点续跑）
*/
@XxlJob("syncCustomerData")
public void syncCustomerData() {
customerSyncEngine.runSync();
}
}3、实现类
java
package com.ybt.center.platform.service.system;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.ybt.center.common.entity.pojo.ApiResponse;
import com.ybt.center.platform.config.CustomerSyncProperties;
import com.ybt.center.platform.entity.POJO.generator.Customer;
import com.ybt.center.platform.entity.POJO.generator.CustomerSyncLog;
import com.ybt.center.platform.entity.POJO.generator.Departments;
import com.ybt.center.platform.infrastructure.svc.feignclient.PortraitFeignClient;
import com.ybt.center.platform.infrastructure.svc.feignclient.dto.CustomerDO;
import com.ybt.center.platform.infrastructure.svc.feignclient.vo.CustomerPageQuery;
import com.ybt.center.platform.service.generator.CustomerSyncLogService;
import com.xxl.job.core.context.XxlJobHelper;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@Service
public class CustomerSyncEngine {
public static final String LOCK_KEY = "customer:sync:lock";
private static final int STATUS_RUNNING = 0;
private static final int STATUS_SUCCESS = 1;
private static final int STATUS_FAILED = 2;
private static final int STATUS_TIMEOUT = 3;
@Resource
private PortraitFeignClient portraitFeignClient;
@Resource
private DepartmentService departmentService;
@Resource
private CustomerSyncLogService customerSyncLogService;
@Resource
private CustomerSyncPageWriter pageWriter;
@Resource
private CustomerSyncProperties props;
@Resource
private RedissonClient redissonClient;
// ==================== 两个对外入口 ====================
/** 主同步（@XxlJob("syncCustomerData")）。 */
public void runSync() {
RLock lock = redissonClient.getLock(LOCK_KEY);
// tryLock 不指定 leaseTime → Redisson watchdog 自动续期
if (!lock.tryLock()) {
log.info("客户同步已有实例在跑，本次跳过");
return;
}
try {
SyncContext ctx = initOrResumeContext();
runBatch(ctx);
} finally {
if (lock.isHeldByCurrentThread()) {
lock.unlock();
}
}
}
/** 补偿（@XxlJob("compensateCustomerSync")）。 */
public void runCompensation() {
// 0. 无需持锁的巡检：疑似卡死（心跳超时但锁可能仍被占用）→ 只告警
alertStaleRunning();
RLock lock = redissonClient.getLock(LOCK_KEY);
if (!lock.tryLock()) {
log.info("主同步任务正在运行，补偿任务跳过");
return;
}
try {
// 1. 持锁后：孤儿 RUNNING（进程已死）→ TIMEOUT
markStaleRunningAsTimeout();
// 2. 捡一个待补偿批次
CustomerSyncLog pending = findPendingCompensation();
if (pending == null) {
return;
}
int retry = pending.getRetryCount() == null ? 0 : pending.getRetryCount();
customerSyncLogService.update(new LambdaUpdateWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getId, pending.getId())
.set(CustomerSyncLog::getRetryCount, retry + 1)
.set(CustomerSyncLog::getSyncStatus, STATUS_RUNNING)
.set(CustomerSyncLog::getTriggerSource, 2)
.set(CustomerSyncLog::getUpdateTime, LocalDateTime.now()));
pending.setRetryCount(retry + 1);
pending.setSyncStatus(STATUS_RUNNING);
pending.setTriggerSource(2);
runBatch(SyncContext.from(pending));
} finally {
if (lock.isHeldByCurrentThread()) {
lock.unlock();
}
}
}
// ==================== 上下文 ====================
private SyncContext initOrResumeContext() {
CustomerSyncLog resumable = customerSyncLogService.getOne(new LambdaQueryWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getSyncStatus, STATUS_RUNNING)
.orderByDesc(CustomerSyncLog::getId)
.last("LIMIT 1"));
if (resumable != null) {
log.info("发现未完成批次, batchId={}, 从游标续跑", resumable.getSyncBatchId());
return SyncContext.from(resumable);
}
CustomerSyncLog fresh = new CustomerSyncLog();
fresh.setSyncBatchId(UUID.randomUUID().toString().replace("-", ""));
fresh.setSyncStatus(STATUS_RUNNING);
fresh.setTotalSynced(0);
fresh.setRetryCount(0);
fresh.setTriggerSource(1);
fresh.setStartTime(LocalDateTime.now());
fresh.setHeartbeatTime(LocalDateTime.now());
fresh.setCreateTime(LocalDateTime.now());
fresh.setUpdateTime(LocalDateTime.now());
customerSyncLogService.save(fresh);
log.info("新建同步批次, batchId={}", fresh.getSyncBatchId());
return SyncContext.from(fresh);
}
// ==================== 主循环（并行写 + 有序提交） ====================
private void runBatch(SyncContext ctx) {
ThreadPoolExecutor writerPool = newWriterPool();
AtomicBoolean failFast = new AtomicBoolean(false);
AtomicInteger pendingWrites = new AtomicInteger(0);
BlockingQueue<SyncResult> results = new LinkedBlockingQueue<>();
TreeMap<Integer, SyncResult> pendingResults = new TreeMap<>();
AtomicInteger committedPageNo = new AtomicInteger(ctx.pageNo - 1);
Map<String, Departments> deptMap = loadDepartmentMap();
try {
int pageNo = ctx.pageNo;
while (!failFast.get()) {
ctx.pageNo = pageNo;
List<CustomerDO> page = fetchPage(ctx.lastCustomerNo);
if (page == null) {
drainAll(results, pendingResults, committedPageNo, ctx, failFast);
recordFail(ctx, "第" + pageNo + "页拉取重试" + props.getMaxRetry() + "次仍失败");
return;
}
if (page.isEmpty()) {
break;
}
String pageLast = page.get(page.size() - 1).getCustomerNo();
ctx.lastCustomerNo = pageLast; // 拉取游标立即推进（下次拉取不重复）
List<Customer> customers = convert(page, deptMap);
pendingWrites.incrementAndGet();
submitWrite(writerPool, pageNo, pageLast, customers, results, pendingWrites, failFast);
drainInOrder(results, pendingResults, committedPageNo, ctx, failFast);
if (failFast.get()) {
return;
}
backpressure(results, pendingResults, committedPageNo, pendingWrites, ctx, failFast);
if (failFast.get()) {
return;
}
if (page.size() < props.getFetchSize()) {
break;
}
pageNo++;
}
awaitPending(pendingWrites);
drainAll(results, pendingResults, committedPageNo, ctx, failFast);
if (!failFast.get()) {
if (pendingWrites.get() > 0) {
recordFail(ctx, "收尾等待在途写入超时, pending=" + pendingWrites.get());
} else {
markSuccess(ctx);
}
}
} finally {
writerPool.shutdownNow();
}
}
private ThreadPoolExecutor newWriterPool() {
int n = props.getWriterThreads();
return new ThreadPoolExecutor(n, n, 60L, TimeUnit.SECONDS,
new LinkedBlockingQueue<>(), new NamedThreadFactory("customer-sync-writer-"),
new ThreadPoolExecutor.CallerRunsPolicy());
}
private void submitWrite(ThreadPoolExecutor writerPool, int pageNo, String pageLast, List<Customer> customers,
BlockingQueue<SyncResult> results, AtomicInteger pendingWrites, AtomicBoolean failFast) {
final int pn = pageNo;
List<Customer> snapshot = new ArrayList<>(customers);
writerPool.submit(() -> {
try {
if (failFast.get()) {
return;
}
int n = pageWriter.savePageWithFreeze(snapshot);
results.put(SyncResult.success(pn, pageLast, n));
} catch (Throwable t) {
log.error("第{}页写入异常", pn, t);
try {
results.put(SyncResult.error(pn, pageLast, t));
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
} finally {
pendingWrites.decrementAndGet();
}
});
}
/** 按序消费已完成结果，推进游标 + 心跳（只在主线程调用）。 */
private void drainInOrder(BlockingQueue<SyncResult> results, TreeMap<Integer, SyncResult> pending,
AtomicInteger committedPageNo, SyncContext ctx, AtomicBoolean failFast) {
SyncResult r;
while ((r = results.poll()) != null) {
if (!r.poison) {
pending.put(r.pageNo, r);
}
}
int next = committedPageNo.get() + 1;
while (pending.containsKey(next)) {
SyncResult res = pending.remove(next);
if (res.error != null) {
failFast.set(true);
recordFail(ctx, "第" + next + "页写入失败: " + res.error.getMessage());
return;
}
ctx.totalSynced += res.successCount;
ctx.committedCursor = res.lastCustomerNo;
committedPageNo.set(next);
updateProgress(ctx);
next++;
}
}
private void backpressure(BlockingQueue<SyncResult> results, TreeMap<Integer, SyncResult> pending,
AtomicInteger committedPageNo, AtomicInteger pendingWrites,
SyncContext ctx, AtomicBoolean failFast) {
while (pendingWrites.get() >= props.getMaxInflight() && !failFast.get()) {
drainInOrder(results, pending, committedPageNo, ctx, failFast);
if (failFast.get() || pendingWrites.get() < props.getMaxInflight()) {
break;
}
try {
SyncResult r = results.poll(500, TimeUnit.MILLISECONDS);
if (r != null && !r.poison) {
pending.put(r.pageNo, r);
}
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
failFast.set(true);
}
}
}
private void drainAll(BlockingQueue<SyncResult> results, TreeMap<Integer, SyncResult> pending,
AtomicInteger committedPageNo, SyncContext ctx, AtomicBoolean failFast) {
SyncResult r;
while ((r = results.poll()) != null) {
if (!r.poison) {
pending.put(r.pageNo, r);
}
}
drainInOrder(results, pending, committedPageNo, ctx, failFast);
}
private void awaitPending(AtomicInteger pendingWrites) {
long deadline = System.currentTimeMillis() + 30_000L;
while (pendingWrites.get() > 0 && System.currentTimeMillis() < deadline) {
try {
Thread.sleep(50);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
break;
}
}
if (pendingWrites.get() > 0) {
log.warn("等待在途写入超时, pending={}", pendingWrites.get());
}
}
// ==================== 拉取 / 转换 ====================
private List<CustomerDO> fetchPage(String lastCustomerNo) {
CustomerPageQuery q = new CustomerPageQuery();
q.setLastCustomerNo(lastCustomerNo);
q.setLimit(props.getFetchSize());
String lastError = null;
for (int i = 0; i < props.getMaxRetry(); i++) {
try {
ApiResponse<List<CustomerDO>> resp = portraitFeignClient.selectPage(q);
if (resp != null && Boolean.TRUE.equals(resp.getSuccess())) {
return resp.getData() != null ? resp.getData() : Collections.emptyList();
}
lastError = "返回失败: " + resp;
} catch (Exception e) {
lastError = e.toString();
}
if (i < props.getMaxRetry() - 1) {
sleepQuietly(props.getRetryDelayMs());
}
}
log.error("拉取客户页重试{}次仍失败, cursor={}, error={}", props.getMaxRetry(), lastCustomerNo, lastError);
return null;
}
private Map<String, Departments> loadDepartmentMap() {
return departmentService.list(new QueryWrapper<Departments>()
.lambda().isNotNull(Departments::getAddrBankCode)).stream()
.collect(Collectors.toMap(Departments::getAddrBankCode, Function.identity(), (a, b) -> a));
}
private List<Customer> convert(List<CustomerDO> page, Map<String, Departments> deptMap) {
List<Customer> out = new ArrayList<>(page.size());
LocalDateTime now = LocalDateTime.now();
for (CustomerDO d : page) {
Customer c = new Customer();
c.setCustomerNo(d.getCustomerNo());
c.setCustomerName(d.getName());
c.setSex(d.getGender());
c.setBirthday(d.getBirthDay() != null ? d.getBirthDay().toLocalDate() : null);
c.setOrganizationCode(d.getBankName());
if (StringUtils.isNotBlank(d.getBankName()) && deptMap.containsKey(d.getBankName())) {
c.setOrganizationName(deptMap.get(d.getBankName()).getName());
c.setPermDeptId(deptMap.get(d.getBankName()).getId());
}
c.setUserStatus(parseStatus(d.getStatus()));
c.setIdCard(d.getIdCard());
c.setPhone(d.getPhone());
c.setVipLevel(d.getVipLevel());
if (d.getGraylistFlag() != null && d.getGraylistFlag() == 1) {
c.setUserStatus(2);
}
c.setCreateTime(now);
c.setUpdateTime(now);
out.add(c);
}
return out;
}
private Integer parseStatus(String status) {
if (status == null) {
return 1;
}
try {
return Integer.parseInt(status.trim());
} catch (Exception e) {
return 1;
}
}
// ==================== 进度落库 ====================
private void updateProgress(SyncContext ctx) {
customerSyncLogService.update(new LambdaUpdateWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getId, ctx.syncLog.getId())
.set(CustomerSyncLog::getLastCustomerNo, ctx.committedCursor)
.set(CustomerSyncLog::getTotalSynced, ctx.totalSynced)
.set(CustomerSyncLog::getHeartbeatTime, LocalDateTime.now())
.set(CustomerSyncLog::getErrorMsg, null)
.set(CustomerSyncLog::getUpdateTime, LocalDateTime.now()));
}
private void markSuccess(SyncContext ctx) {
customerSyncLogService.update(new LambdaUpdateWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getId, ctx.syncLog.getId())
.set(CustomerSyncLog::getLastCustomerNo, ctx.committedCursor)
.set(CustomerSyncLog::getTotalSynced, ctx.totalSynced)
.set(CustomerSyncLog::getSyncStatus, STATUS_SUCCESS)
.set(CustomerSyncLog::getEndTime, LocalDateTime.now())
.set(CustomerSyncLog::getUpdateTime, LocalDateTime.now()));
XxlJobHelper.log("同步完成 batchId={}, total={}", ctx.batchId, ctx.totalSynced);
log.info("同步完成 batchId={}, total={}", ctx.batchId, ctx.totalSynced);
}
private void recordFail(SyncContext ctx, String reason) {
customerSyncLogService.update(new LambdaUpdateWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getId, ctx.syncLog.getId())
.set(CustomerSyncLog::getLastCustomerNo, ctx.committedCursor)
.set(CustomerSyncLog::getTotalSynced, ctx.totalSynced)
.set(CustomerSyncLog::getSyncStatus, STATUS_FAILED)
.set(CustomerSyncLog::getErrorMsg, reason)
.set(CustomerSyncLog::getEndTime, LocalDateTime.now())
.set(CustomerSyncLog::getUpdateTime, LocalDateTime.now()));
XxlJobHelper.log("ERROR: {}", reason);
log.error("同步失败 batchId={}, reason={}", ctx.batchId, reason);
}
// ==================== 补偿辅助 ====================
private void alertStaleRunning() {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(props.getHeartbeatTimeoutMin());
customerSyncLogService.list(new LambdaQueryWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getSyncStatus, STATUS_RUNNING)
.isNotNull(CustomerSyncLog::getHeartbeatTime))
.stream().filter(l -> l.getHeartbeatTime().isBefore(cutoff))
.forEach(l -> log.warn("疑似卡死: batchId={} 心跳 {} 已超 {} 分钟无进展",
l.getSyncBatchId(), l.getHeartbeatTime(), props.getHeartbeatTimeoutMin()));
}
private void markStaleRunningAsTimeout() {
LocalDateTime cutoff = LocalDateTime.now().minusMinutes(props.getHeartbeatTimeoutMin());
customerSyncLogService.list(new LambdaQueryWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getSyncStatus, STATUS_RUNNING)
.isNotNull(CustomerSyncLog::getHeartbeatTime))
.stream().filter(l -> l.getHeartbeatTime().isBefore(cutoff))
.forEach(l -> {
customerSyncLogService.update(new LambdaUpdateWrapper<CustomerSyncLog>()
.eq(CustomerSyncLog::getId, l.getId())
.set(CustomerSyncLog::getSyncStatus, STATUS_TIMEOUT)
.set(CustomerSyncLog::getUpdateTime, LocalDateTime.now()));
log.warn("孤儿 RUNNING 批次置 TIMEOUT, batchId={}", l.getSyncBatchId());
});
}
private CustomerSyncLog findPendingCompensation() {
return customerSyncLogService.getOne(new LambdaQueryWrapper<CustomerSyncLog>()
.in(CustomerSyncLog::getSyncStatus, STATUS_FAILED, STATUS_TIMEOUT)
.and(w -> w.isNull(CustomerSyncLog::getRetryCount)
.or().lt(CustomerSyncLog::getRetryCount, props.getMaxCompensateRetry()))
.orderByAsc(CustomerSyncLog::getStartTime)
.last("LIMIT 1"));
}
private void sleepQuietly(long millis) {
try {
Thread.sleep(millis);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
}
// ==================== 内部类 ====================
private static class SyncContext {
String batchId;
String lastCustomerNo = ""; // 拉取游标
String committedCursor = ""; // 提交游标（落库）
int pageNo = 1;
int totalSynced = 0;
CustomerSyncLog syncLog;
static SyncContext from(CustomerSyncLog l) {
SyncContext c = new SyncContext();
c.batchId = l.getSyncBatchId();
c.lastCustomerNo = StringUtils.defaultString(l.getLastCustomerNo());
c.committedCursor = StringUtils.defaultString(l.getLastCustomerNo());
c.totalSynced = l.getTotalSynced() == null ? 0 : l.getTotalSynced();
c.pageNo = 1; // 仅日志展示用，续跑正确性靠游标
c.syncLog = l;
return c;
}
}
private static class SyncResult {
final int pageNo;
final String lastCustomerNo;
final int successCount;
final Throwable error;
final boolean poison;
SyncResult(int pageNo, String lastCustomerNo, int successCount, Throwable error, boolean poison) {
this.pageNo = pageNo;
this.lastCustomerNo = lastCustomerNo;
this.successCount = successCount;
this.error = error;
this.poison = poison;
}
static SyncResult success(int pageNo, String lastCustomerNo, int successCount) {
return new SyncResult(pageNo, lastCustomerNo, successCount, null, false);
}
static SyncResult error(int pageNo, String lastCustomerNo, Throwable error) {
return new SyncResult(pageNo, lastCustomerNo, 0, error, false);
}
}
private static class NamedThreadFactory implements ThreadFactory {
private final String prefix;
private final AtomicInteger counter = new AtomicInteger(1);
NamedThreadFactory(String prefix) {
this.prefix = prefix;
}
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r, prefix + counter.getAndIncrement());
t.setDaemon(false);
return t;
}
}
}4、配置类
java
package com.ybt.center.platform.config;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* 客户同步配置属性（支持 application.yml 配置化）。
* <p>
* 配置示例：
* <pre>
* customer:
* sync:
* fetch-size: 2000
* batch-size: 500
* writer-threads: 4
* max-writer-threads: 8
* queue-capacity: 20
* max-retry: 3
* retry-delay-ms: 2000
* sync-timeout-hours: 4
* shutdown-await-ms: 1800000
* drain-result-timeout-ms: 300000
* </pre>
*/
@Data
@Component
@ConfigurationProperties(prefix = "customer.sync")
public class CustomerSyncProperties {
/** 每次 Feign 拉取远端数据的条数，默认 2000 */
private int fetchSize = 2000;
/** 每批 VALUES 写入条数，默认 1000 */
private int batchSize = 1000;
/** 写入线程池核心线程数，默认 4（并发化后建议值） */
private int writerThreads = 4;
/** 写入线程池最大线程数，默认 8 */
private int maxWriterThreads = 8;
/** 写入线程池缓冲队列容量，默认 20 */
private int queueCapacity = 20;
/** Feign 单页拉取最大重试次数 */
private int maxRetry = 3;
/** 重试间隔（毫秒） */
private long retryDelayMs = 2000;
/** 断点续传：只续传 N 小时内未完成的同步任务 */
private int syncTimeoutHours = 4;
/** 优雅关闭：等待在途写入完成的最大时长（毫秒），默认 30 分钟 */
private long shutdownAwaitMs = 30 * 60 * 1000L;
/** drainOneResult 等待单页写入结果的超时（毫秒），默认 5 分钟 */
private long drainResultTimeoutMs = 5 * 60 * 1000L;
/** 在途页数上限（背压阈值，≈2×writerThreads） */
private int maxInflight = 8;
/** 心跳超时阈值（分钟），超过则判定卡死/崩溃 */
private int heartbeatTimeoutMin = 5;
/** 失败自动重试（补偿）上限 */
private int maxCompensateRetry = 3;
/** 线程空闲存活时间（秒），默认 60 */
private long keepAliveSeconds = 60L;
/** 线程名前缀，默认 customer-sync-writer- */
private String threadNamePrefix = "customer-sync-writer-";
}5、写库
java
package com.ybt.center.platform.service.system;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.ybt.center.platform.config.CustomerSyncProperties;
import com.ybt.center.platform.entity.POJO.generator.Customer;
import com.ybt.center.platform.entity.POJO.generator.CustomerAuthInfo;
import com.ybt.center.platform.interceptor.CustomerEncryptInterceptor;
import com.ybt.center.platform.mapper.generator.CustomerAuthInfoMapper;
import com.ybt.center.platform.mapper.generator.CustomerMapper;
import lombok.extern.slf4j.Slf4j;
import org.springframework.jdbc.datasource.DataSourceUtils;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
import java.util.stream.Collectors;
@Slf4j
@Service
public class CustomerSyncPageWriter {
@Resource
private CustomerMapper customerMapper;
@Resource
private CustomerAuthInfoMapper customerAuthInfoMapper;
@Resource
private CustomerSyncProperties props;
@Resource
private DataSource dataSource;
@Transactional(rollbackFor = Exception.class)
public int savePageWithFreeze(List<Customer> page) {
// 同步流程的源数据与库内形态一致，整体跳过加解密拦截器（含 writeBatchWithCompare 内的查询）
return CustomerEncryptInterceptor.runWithoutEncrypt(() -> doSavePageWithFreeze(page));
}
/**
* 页级事务写：每批「查已有 → 比对字段 → 分流 insert/update/跳过」，批量 VALUES 写入；
* 最后灰名单冻结清认证（与写入同事务）。
*
* @return 本页实际写入（新增 + 更新）条数
*/
@Transactional(rollbackFor = Exception.class)
public int doSavePageWithFreeze(List<Customer> page) {
int written = 0;
int batchSize = Math.max(1, props.getBatchSize());
for (int i = 0; i < page.size(); i += batchSize) {
int end = Math.min(i + batchSize, page.size());
List<Customer> batch = page.subList(i, end);
written += writeBatchWithCompare(batch);
}
// 灰名单冻结副作用：userStatus==2 的客户清认证字段（与写入同事务）
List<String> frozenNos = page.stream()
.filter(c -> c.getUserStatus() != null && c.getUserStatus() == 2)
.map(Customer::getCustomerNo)
.filter(Objects::nonNull)
.distinct()
.collect(Collectors.toList());
if (!frozenNos.isEmpty()) {
customerAuthInfoMapper.update(null, new LambdaUpdateWrapper<CustomerAuthInfo>()
.in(CustomerAuthInfo::getCustomerNo, frozenNos)
.set(CustomerAuthInfo::getCustomerName, null)
.set(CustomerAuthInfo::getIdCard, null)
.set(CustomerAuthInfo::getFirstWxCertificationTime, null)
.set(CustomerAuthInfo::getFirstAppCertificationTime, null));
log.info("灰名单冻结清认证, 客户数={}", frozenNos.size());
}
return written;
}
/**
* JDBC 原生批量插入（配合 URL 上 rewriteBatchedStatements=true，驱动层合并多值 SQL）。
* 通过 {@link DataSourceUtils} 取当前事务绑定的连接，与 mapper 写入同连接、同事务。
*/
private void jdbcBatchInsert(List<Customer> list) {
String sql = "INSERT INTO CUSTOMER (CUSTOMER_NO, CUSTOMER_NAME, SEX, BIRTHDAY, ORGANIZATION_CODE, "
+ "PHONE, ID_CARD, VIP_LEVEL, USER_STATUS, CREATE_TIME, UPDATE_TIME) "
+ "VALUES (?,?,?,?,?,?,?,?,?,?,?)";
Connection conn = DataSourceUtils.getConnection(dataSource);
try (PreparedStatement ps = conn.prepareStatement(sql)) {
for (Customer c : list) {
ps.setObject(1, c.getCustomerNo());
ps.setObject(2, c.getCustomerName());
ps.setObject(3, c.getSex());
ps.setObject(4, c.getBirthday());
ps.setObject(5, c.getOrganizationCode());
ps.setObject(6, c.getPhone());
ps.setObject(7, c.getIdCard());
ps.setObject(8, c.getVipLevel());
ps.setObject(9, c.getUserStatus());
ps.setObject(10, c.getCreateTime());
ps.setObject(11, c.getUpdateTime());
ps.addBatch();
}
ps.executeBatch();
} catch (Exception e) {
throw new RuntimeException("JDBC批量插入客户失败, 条数=" + list.size(), e);
}
}
/**
* JDBC 原生批量更新（调用前提：行已存在于库中，由 {@link #writeBatchWithCompare(List)} 比对分流保证）。
* 字段语义与原 upsert 保持一致：新值为 null 时保留库中旧值（COALESCE），更新时间取当前时间。
*/
private void jdbcBatchUpdate(List<Customer> list) {
String sql = "UPDATE CUSTOMER SET "
+ "CUSTOMER_NAME = COALESCE(?, CUSTOMER_NAME), "
+ "SEX = COALESCE(?, SEX), "
+ "BIRTHDAY = COALESCE(?, BIRTHDAY), "
+ "ORGANIZATION_CODE = COALESCE(?, ORGANIZATION_CODE), "
+ "ORGANIZATION_NAME = COALESCE(?, ORGANIZATION_NAME), "
+ "PHONE = COALESCE(?, PHONE), "
+ "VIP_LEVEL = COALESCE(?, VIP_LEVEL), "
+ "ID_CARD = COALESCE(?, ID_CARD), "
+ "USER_STATUS = COALESCE(?, USER_STATUS), "
+ "PERM_DEPT_ID = COALESCE(?, PERM_DEPT_ID), "
+ "UPDATE_TIME = NOW() "
+ "WHERE CUSTOMER_NO = ?";
Connection conn = DataSourceUtils.getConnection(dataSource);
try (PreparedStatement ps = conn.prepareStatement(sql)) {
for (Customer c : list) {
ps.setObject(1, c.getCustomerName());
ps.setObject(2, c.getSex());
ps.setObject(3, c.getBirthday());
ps.setObject(4, c.getOrganizationCode());
ps.setObject(5, c.getOrganizationName());
ps.setObject(6, c.getPhone());
ps.setObject(7, c.getVipLevel());
ps.setObject(8, c.getIdCard());
ps.setObject(9, c.getUserStatus());
ps.setObject(10, c.getPermDeptId());
ps.setObject(11, c.getCustomerNo());
ps.addBatch();
}
ps.executeBatch();
} catch (Exception e) {
throw new RuntimeException("JDBC批量更新客户失败, 条数=" + list.size(), e);
}
}
/** 一批内：查已有 → 比对 → 分流 → 批量写入。 */
private int writeBatchWithCompare(List<Customer> batch) {
// 批内按 customerNo 去重（源数据若重复，保留第一条）
Map<String, Customer> srcMap = new LinkedHashMap<>();
for (Customer c : batch) {
if (c.getCustomerNo() != null) {
srcMap.putIfAbsent(c.getCustomerNo(), c);
}
}
if (srcMap.isEmpty()) {
return 0;
}
// 查已有（用于字段比对，判断新增 / 更新 / 跳过）
Map<String, Customer> existMap = customerMapper.selectList(
new LambdaQueryWrapper<Customer>().in(Customer::getCustomerNo, srcMap.keySet()))
.stream()
.collect(Collectors.toMap(Customer::getCustomerNo, Function.identity(), (a, b) -> a));
List<Customer> toInsert = new ArrayList<>();
List<Customer> toUpdate = new ArrayList<>();
for (Customer c : srcMap.values()) {
Customer old = existMap.get(c.getCustomerNo());
if (old == null) {
toInsert.add(c);
} else if (changed(old, c)) {
toUpdate.add(c);
}
// old != null 且未变化 → 跳过，不写库
}
int written = 0;
if (!toInsert.isEmpty()) {
jdbcBatchInsert(toInsert);
written += toInsert.size();
}
if (!toUpdate.isEmpty()) {
jdbcBatchUpdate(toUpdate);
written += toUpdate.size();
}
log.debug("同步批次写入: 新增={}, 更新={}, 跳过={}",
toInsert.size(), toUpdate.size(), srcMap.size() - toInsert.size() - toUpdate.size());
return written;
}
/** 比对参与同步的字段是否发生变化（任一不同即需更新）。 */
private boolean changed(Customer old, Customer c) {
return !Objects.equals(old.getCustomerName(), c.getCustomerName())
|| !Objects.equals(old.getSex(), c.getSex())
|| !Objects.equals(old.getBirthday(), c.getBirthday())
|| !Objects.equals(old.getOrganizationCode(), c.getOrganizationCode())
|| !Objects.equals(old.getOrganizationName(), c.getOrganizationName())
|| !Objects.equals(old.getPhone(), c.getPhone())
|| !Objects.equals(old.getIdCard(), c.getIdCard())
|| !Objects.equals(old.getUserStatus(), c.getUserStatus())
|| !Objects.equals(old.getVipLevel(), c.getVipLevel())
|| !Objects.equals(old.getPermDeptId(), c.getPermDeptId());
}
}三、理解
逻辑就是查询2000条，然后读出来以后 放入线程池中让他们跑分割存库，子线程存库成功后把结果放在安全队列中然后主线程每次循环后都去拿线程安全的队列值， 然后放在排序map中然后去判断map里面有没有符合上批次的下一条，然后有的话就更新数据库批次，如果超过8个还没更新批次就等待一下防止oom。
四、产生疑问解答
1、主线程和子线程如何沟通的
- 只会通过BlockingQueue results安全队列去沟通
2、沟通完成之后如何存库呢
- 沟通完成会从队列拿出来，放入treeMap中，key是页数，那么treeMap取出来的就是最上面的一页
3、何时更改其中最后一条续跑的id呢
- 比如批次跑1234，其中234跑完了，然后1没跑完，那么会存234么，答案不会
- 只以上次最后一页成功的+1去落库更新最后一页。可以查看，pending就是2中的treeMap，然后判断上次最后一页的页码+1
- 比如1+1=2，那么如果第二页跑完了才会更改最后一页和条数，然后下次进来就是2+1=3 如果3没跑完就跳过然后再去跑下一页，然后到这里再去2+3=3判断3是否完成，然后依次加一
java
int next = committedPageNo.get() + 1;
while (pending.containsKey(next)) {4、那会不会oom，有没有限制
- 当然有，会根据当前有没有超过getMaxInflight最大页数然后阻塞自旋等待最大页数降下来
java
while (pendingWrites.get() >= props.getMaxInflight() && !failFast.get()) {5、重复跑的问题
- 比如23跑完，1还没跑完，然后服务器挂掉了，然后数据库有数据，首先这个是我们业务中可以根据某个值判断是否重复，然后是新增还是修改的
6、需要考虑重复跑，要求事务一致性，不能重复跑
- 可以加入手动提交事务，用一个状态安全集合或者啥承接，然后每次进行判断是否可以提交在3、中决定是否提交
7、是否可以封装多套用一套
- 应该可以，但是我比较懒，然后具体封装你可以使用函数接口把取值和插入数据库数据的地方抽取出来
8、 runWithoutEncrypt这个是我绕过一个方法写的，然后你可以直接改
- 给savePageWithFreeze改成doSavePageWithFreeze
五、报错重跑补偿
java
package com.ybt.center.platform.job;
import com.xxl.job.core.handler.annotation.XxlJob;
import com.ybt.center.platform.service.system.CustomerSyncEngine;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
@Slf4j
@Component
public class CustomerSyncCompensateJob {
@Resource
private CustomerSyncEngine customerSyncEngine;
/**
* 补偿任务：查表捡失败/超时批次自动续跑（无需人工填批次号）
*/
@XxlJob("compensateCustomerSync")
public void compensateCustomerSync() {
customerSyncEngine.runCompensation();
}
}