总体思路是，主节点接收到任务请求，将根据任务情况拆分成多个任务块，将任务块标识的主键放入redis。发送redis消息，等待其他节点运行完毕，结束处理。接收到信息的节点注册本节点信息到redis、开启多线程、获取任务块、执行任务、结束处理。
1、主节点接收任务请求
java
@Override
public void executeTaskInfo(PrepareDTO prepareDTO) {
//异常标记
String taskInfo = prepareDTO.getTaskId();
//任务分组状态
String taskStatus = "";
try {
log.info("数据准备任务并设定任务执行状态,{}", prepareDTO);
this.dataPrepareBo.doStartGroupJobInfo(prepareDTO);
//给redis集合中放计算对象
log.info("开始放入计算任务:{}", prepareDTO);
boolean getTaskFlag = this.dataPrepareBo.pushCalculationObject(prepareDTO);
if (!getTaskFlag) {
taskStatus = String.format("没有获取数据或计划已取消,%s", taskInfo);
log.error(taskStatus);
throw new Exception(taskStatus);
}
//发消息执行缓存中任务
log.info("发消息执行任务:{}", prepareDTO);
sendMessage(prepareDTO);
//等待任务执行完毕
log.info("等待任务执行结果");
taskStatus = this.getGroupUpLoadTaskFinsh(prepareDTO);
} catch (Exception e) {//捕获日志
e.printStackTrace();
taskStatus = "获取任务状态异常" + e;
log.info(taskStatus);
dataPrepareBo.putExceptionMsg2Cache(taskInfo, "数据准备分发计算任务线程异常:" + taskStatus);
} finally {
//做任务结束处理
this.doGroupTaskFinshpPocess(prepareDTO, taskStatus);
}
}2，发送消息
java
@Override
public void sendMessage(String topic, String msg) {
this.redisTemplate.convertAndSend(topic, msg);
}3，节点接收任务，并执行
java
public void doUpLoadTask(String msg) throws Exception {
log.info("开始执行明细任务{}" + msg);
String taskId = this.getTaskId(msg);
try {
Object cancelFlag = this.redisTemplate.opsForValue().get(String.format(EngineConstant.JOB_CANCEL_FLAG, taskId));
if(cancelFlag != null && "1".equals(cancelFlag.toString())){
log.info("本次任务已取消");
return;
}
//上传本机执行信息到redis
this.cacheBo.initGroupUpLoadTaskStats(taskId,ENGINE_DISTRIBUTION_RUNNING.getKey());
//从缓存获取任务,获取任务后启线程执行任务。如果没获取到任务，则本节点任务执行完毕
//循环获取任务
this.groupTaskProcessBO.doGroupTaskProcess(taskId, null);
//处理结束
this.cacheBo.finishGroupUpLoadTaskStats(taskId,ENGINE_DISTRIBUTION_RUNNING.getKey());
} catch (Exception e) {
//记录日志
taskUpldExeLogCDTO.setRunStas("-1");
String exceptionInfo = this.taskLogUtils.getExceptionInfo(e) ;
taskUpldExeLogCDTO.setAbnInfo(exceptionInfo);
throw e;
} finally {
//记录日志
taskUpldExeLogCDTO.setEndtime(DateUtil.getCurrentDate());
if("-1".equals(taskUpldExeLogCDTO.getRunStas())){//异常结束
this.taskLogUtils.sendLogInfo(taskUpldExeLogCDTO,"执行上传任务异常");
} else {//正常结束
taskUpldExeLogCDTO.setRunStas("1");
this.taskLogUtils.sendLogInfo(taskUpldExeLogCDTO,"执行上传任务正常");
}
}
}4，开启线程执行任务
java
@Override
public CalculationDTO doGroupTaskProcess(String taskId, TaskUpldExeLogCDTO taskUpldExeLogCDTO) throws Exception {
List<Future> futureList = new ArrayList<>();
//开始执行明细任务处理
ThreadPoolTaskExecutor taskTransferExecutor = ToolUtil.getExecutor("engine-file-tasks-pool-", Math.min(parallelProcessNum,10), 8);
ExecutorListHolder.putThreadPool(String.format(GroupConstant.PREPARE_ENGINE_POOL,taskId), taskTransferExecutor.getThreadPoolExecutor());
for(int i = 0 ; i < parallelProcessNum ; i++) {
DoGroupUpLoadTaskThread doGroupUpLoadTaskThread = new DoGroupUpLoadTaskThread(taskId
, redisTemplate, calculationBo, null, null);
Future<?> future = taskTransferExecutor.submit(doGroupUpLoadTaskThread);
futureList.add(future);
}
if (!CollectionUtil.isEmpty(futureList)) {
futureList.forEach(f -> {
try {
f.get(GroupTaskProcessBOImpl.maxTime, TimeUnit.SECONDS);
} catch (Exception e) {
e.printStackTrace();
}
});
}
log.info("本节点执行分组任务执行完毕{}", taskId + ":" + GroupConstant.IDENTITY);
return null;
}5，线程执行明细
java
@Override
public ResponseDTO call() throws Exception {
//执行任务
while(true) {
FilterTableUniqueDTO filterTableUniqueDTO = (FilterTableUniqueDTO)this.redisTemplate.opsForList().leftPop(String.format(ENGINE_FILTERTABLEUNIQUE_QUEUE.getKey(), taskId));
log.debug("取出任务:" + filterTableUniqueDTO);
if(null == filterTableUniqueDTO) {
break ;
}
long lastNum = this.redisTemplate.opsForList().size(String.format(ENGINE_FILTERTABLEUNIQUE_QUEUE.getKey(), taskId));
log.info("生成文件剩余任务数量:" + lastNum);
// 处理任务
calculationBo.GenerateFile(filterTableUniqueDTO, taskUpldDetlLogCDTO);
}
return null;
}以上是主要入口总体思路涉及代码，详细实现整理起来涉及内容比较繁多，将在第二部分分享。