百万数据量修改数据思路及方法

场景

公司定时任务因数据量过大运行时间太久,大约3-4个小时,需要优化代码。数据量一旦变大,普通的修改操作也会变得复杂。

原代码

OpsPriceServiceImpl

java 复制代码
@Override
public ExpireProductPriceRefeshResponse refeshExpireProductPrice(ExpireProductPriceRefeshRequest request) {
    if (log.isDebugEnabled()) {
        log.debug("begin refeshExpireProductPrice request: {}", JsonUtil.toJson(request));
    }
    ExpireProductPriceRefeshResponse response = new ExpireProductPriceRefeshResponse();
    try {
        request.validate(Constant.SUB_SYSTEM, ErpExceptionType.VCE13008);
        Long tenantNumId = request.getTenantNumId();
        Long dataSign = request.getDataSign();
        Date today = DateUtils.parse(DateUtils.format(new Date()));
        List<MDMS_P_PRODUCT_SHOP_PRICE_LOG> shopLogs = mdmsPProductShopPriceLogDao.queryProductShopLogByEndDay(tenantNumId, dataSign, today);
        if (CollectionUtils.isNotEmpty(shopLogs)) {
            CompletableFuture.runAsync(() -> {
                for (MDMS_P_PRODUCT_SHOP_PRICE_LOG log : shopLogs) {
                    mdmsPProductShopDao.updateShopPrice(tenantNumId, dataSign, log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(), log.getITEM_NUM_ID(), ValidatorUtils.isNullOrZero(log.getOLD_PRICE()) ? null:log.getOLD_PRICE(), 1L, log.getPRICE_TYPE());
                    mdmsPProductShopPriceLogDao.updateShopLogNotValid(tenantNumId,dataSign,1L,log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(),log.getITEM_NUM_ID(),log.getSERIES());
                }
            }, opsThreadPoolExecutor).exceptionally(ex -> {
                log.error("refeshExpireProductPrice failed, cause: " + ex.getMessage());
                return null;
            });
        }
    } catch (Exception ex) {
        ExceptionUtil.processException(ex, response);
    }
    if (log.isDebugEnabled()) {
        log.debug("end refeshExpireProductPrice response:{}", JsonUtil.toJson(response));
    }
    return response;
}

mdmsPProductShopDao

java 复制代码
public void updateShopPrice(Long tenantNumId, Long dataSign, String cortNumId, String subUnitNumId, String itemNumId,
                            Double price, Long usrNumId, int priceType) {
    StringBuilder sb = new StringBuilder();
    sb.append("update mdms_p_product_shop set last_updtme = now() ");
    if (priceType == 501) {
        sb.append(",retail_price =?");
    } else if (priceType == 502) {
        sb.append(",member_price =?");
    } else if (priceType == 503) {
        sb.append(",member_day_price =?");
    } else if (priceType == 504) {
        sb.append(",tiny_price =?");
    } else if (priceType == 505) {
        sb.append(",health_care_price =?");
    } else if (priceType == 506) {
        sb.append(",floor_price =?");
    }
    sb.append(",last_update_user_id=?  where tenant_num_id = ? and data_sign = ? and cort_num_id=? and sub_unit_num_id = ? and item_num_id = ?");
    int row = jdbcTemplate.update(sb.toString(),
            new Object[]{price, usrNumId, tenantNumId, dataSign, cortNumId, subUnitNumId, itemNumId});
    if (row <= 0) {
        throw new DatabaseOperateException(Constant.SUB_SYSTEM, ErpExceptionType.DOE33008,
                "更新零售价价格失败!门店号:" + subUnitNumId + " 商品编号:" + itemNumId);
    }
}

mdmsPProductShopPriceLogDao

java 复制代码
public void updateShopLogNotValid(Long tenantNumId, Long dataSign,Long usrNumId,String cortNumId,
                                String subUnitNumId, String itemNumId , String  series) {
    String sql = "update mdms_p_product_shop_price_log set is_valid =0,last_updtme = now(),last_update_user_id=? "
            + " where tenant_num_id = ? and data_sign = ?  and cort_num_id=? and sub_unit_num_id = ? and item_num_id = ?  "
            + " and  series=?  and cancelsign='N'";
    int row = jdbcTemplate.update(sql,
            new Object[]{ usrNumId,tenantNumId, dataSign,cortNumId, subUnitNumId, itemNumId,series});
}

问题

由以上代码可见,代码虽使用CompletableFuture.runAsync()进行异步执行,但下方通过增强for循环单个进行修改操作,且调用的方法通过了if else进行了多次判断,导致速度缓慢。

思路

第一个想法

通过将if else进行舍弃将其中sql拼接不通过priceType进行判断,认真读代码后,发现是进行不同字段的修改,放弃。

第二个想法

if else改为switch,同时将OpsPriceServiceImpl中的增强for循环改为stream流形式。当数据量不超过10万条数据时,增强for循环速度快,但超过时stream流形式更为快速。
原OpsPriceServiceImpl

java 复制代码
  for (MDMS_P_PRODUCT_SHOP_PRICE_LOG log : shopLogs) {
      mdmsPProductShopDao.updateShopPrice(tenantNumId, dataSign, log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(), log.getITEM_NUM_ID(), ValidatorUtils.isNullOrZero(log.getOLD_PRICE()) ? null:log.getOLD_PRICE(), 1L, log.getPRICE_TYPE());
      mdmsPProductShopPriceLogDao.updateShopLogNotValid(tenantNumId,dataSign,1L,log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(),log.getITEM_NUM_ID(),log.getSERIES());
  }

改为

java 复制代码
   shopLogs.stream().forEach(log->{
      mdmsPProductShopDao.updateShopPrice(tenantNumId, dataSign, log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(), log.getITEM_NUM_ID(), ValidatorUtils.isNullOrZero(log.getOLD_PRICE()) ? null:log.getOLD_PRICE(), 1L, log.getPRICE_TYPE());
      mdmsPProductShopPriceLogDao.updateShopLogNotValid(tenantNumId,dataSign,1L,log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(),log.getITEM_NUM_ID(),log.getSERIES());
  });

后发现stream流循环通过parallelStream可以进行并行操作,后将代码改为

java 复制代码
   shopLogs.parallelStream().forEach(log->{
      mdmsPProductShopDao.updateShopPrice(tenantNumId, dataSign, log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(), log.getITEM_NUM_ID(), ValidatorUtils.isNullOrZero(log.getOLD_PRICE()) ? null:log.getOLD_PRICE(), 1L, log.getPRICE_TYPE());
      mdmsPProductShopPriceLogDao.updateShopLogNotValid(tenantNumId,dataSign,1L,log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(),log.getITEM_NUM_ID(),log.getSERIES());
  });

原updateShopPrice方法

java 复制代码
if (priceType == 501) {
    sb.append(",retail_price =?");
} else if (priceType == 502) {
    sb.append(",member_price =?");
} else if (priceType == 503) {
    sb.append(",member_day_price =?");
} else if (priceType == 504) {
    sb.append(",tiny_price =?");
} else if (priceType == 505) {
    sb.append(",health_care_price =?");
} else if (priceType == 506) {
    sb.append(",floor_price =?");
}

改为

java 复制代码
switch (priceType) {
    case 501:
        sb.append(",retail_price =?");
        break;
    case 502:
        sb.append(",member_price =?");
        break;
    case 503:
        sb.append(",member_day_price =?");
        break;
    case 504:
        sb.append(",tiny_price =?");
        break;
    case 505:
        sb.append(",health_care_price =?");
        break;
    case 506:
        sb.append(",floor_price =?");
        break;
}

第三个想法

虽效率提高了部分,但感觉还是不够满意。原代码修改操作通过jdbcTemplate.update()进行单条处理,决定通过jdbcTemplate.batchUpdate()进行批量处理。因每循环一次需要对两个方法进行修改操作,可能会出现数据不统一问题,将这部分代码加入事务,出现异常后回滚(masterDataTransactionManager.getTransaction(TransactionUtil.newTransactionDefinition(300)))。

修改后全部代码
OpsPriceServiceImpl

java 复制代码
@Override
public ExpireProductPriceRefeshResponse refeshExpireProductPrice(ExpireProductPriceRefeshRequest request) {
    if (log.isDebugEnabled()) {
        log.debug("begin refeshExpireProductPrice request: {}", JsonUtil.toJson(request));
    }
    ExpireProductPriceRefeshResponse response = new ExpireProductPriceRefeshResponse();
    try {
        request.validate(Constant.SUB_SYSTEM, ErpExceptionType.VCE13008);
        Long tenantNumId = request.getTenantNumId();
        Long dataSign = request.getDataSign();
        Date today = DateUtils.parse(DateUtils.format(new Date()));
        List<MDMS_P_PRODUCT_SHOP_PRICE_LOG> shopLogs = mdmsPProductShopPriceLogDao.queryProductShopLogByEndDay(tenantNumId, dataSign, today);
        if (CollectionUtils.isNotEmpty(shopLogs)) {
            TransactionStatus status = masterDataTransactionManager.getTransaction(TransactionUtil.newTransactionDefinition(300));
            try {
                CompletableFuture.runAsync(() -> {
                    if (shopLogs.size() > 2000) {
                        List<List<MDMS_P_PRODUCT_SHOP_PRICE_LOG>> partition = Lists.partition(shopLogs, 2000);
                        for (List<MDMS_P_PRODUCT_SHOP_PRICE_LOG> mdms_p_product_shop_price_logs : partition) {
                            mdmsPProductShopDao.batchUpdateShopPrice(tenantNumId, dataSign, mdms_p_product_shop_price_logs);
                            mdmsPProductShopPriceLogDao.batchUpdateShopLogNotValid(tenantNumId, dataSign, mdms_p_product_shop_price_logs);
                        }
                    } else {
                        mdmsPProductShopDao.batchUpdateShopPrice(tenantNumId, dataSign, shopLogs);
                        mdmsPProductShopPriceLogDao.batchUpdateShopLogNotValid(tenantNumId, dataSign, shopLogs);
                    }
                }, opsThreadPoolExecutor).exceptionally(ex -> {
                    log.error("refeshExpireProductPrice failed, cause: " + ex.getMessage());
                    return null;
                });
                masterDataTransactionManager.commit(status);
            } catch (Exception ex) {
                masterDataTransactionManager.rollback(status);
                throw ex;
            }
        }

    } catch (Exception ex) {
        ExceptionUtil.processException(ex, response);
    }
    if (log.isDebugEnabled()) {
        log.debug("end refeshExpireProductPrice response:{}", JsonUtil.toJson(response));
    }
    return response;
}

mdmsPProductShopDao

java 复制代码
    public void batchUpdateShopPriceCase(Long tenantNumId, Long dataSign, StringBuilder sql, List<MDMS_P_PRODUCT_SHOP_PRICE_LOG> mdms_p_product_shop_price_logs) {
        sql.append(",last_update_user_id=?  where tenant_num_id = ? and data_sign = ? and cort_num_id=? and sub_unit_num_id = ? and item_num_id = ?");
        List<Object[]> batchArgs = new ArrayList<>();
        for (MDMS_P_PRODUCT_SHOP_PRICE_LOG priceLog : mdms_p_product_shop_price_logs) {
            Object[] object = new Object[]{ValidatorUtils.isNullOrZero(priceLog.getOLD_PRICE()) ? null : priceLog.getOLD_PRICE(), 1L, tenantNumId, dataSign, priceLog.getCORT_NUM_ID(), priceLog.getSUB_UNIT_NUM_ID(), priceLog.getITEM_NUM_ID()};
            batchArgs.add(object);
        }
        int[] ints = jdbcTemplate.batchUpdate(String.valueOf(sql), batchArgs);
        int sum = DaoUtil.sum(ints);
        if (sum != batchArgs.size()) {
            throw new DatabaseOperateException(Constant.SUB_SYSTEM, ErpExceptionType.DOE33008,
                    "批量更新零售价价格失败!");
        }
    }

    public void batchUpdateShopPrice(Long tenantNumId, Long dataSign, List<MDMS_P_PRODUCT_SHOP_PRICE_LOG> mdms_p_product_shop_price_logs) {
        Map<Integer, List<MDMS_P_PRODUCT_SHOP_PRICE_LOG>> collectMap = mdms_p_product_shop_price_logs.stream().collect(Collectors.groupingBy(MDMS_P_PRODUCT_SHOP_PRICE_LOG::getPRICE_TYPE));
        for (Integer integer : collectMap.keySet()) {
            StringBuilder sb = new StringBuilder();
            sb.append("update mdms_p_product_shop set last_updtme = now() ");
            switch (integer) {
                case 501:
                    sb.append(",retail_price =?");
                    batchUpdateShopPriceCase(tenantNumId, dataSign, sb, collectMap.get(integer));
                    break;
                case 502:
                    sb.append(",member_price =?");
                    batchUpdateShopPriceCase(tenantNumId, dataSign, sb, collectMap.get(integer));
                    break;
                case 503:
                    sb.append(",member_day_price =?");
                    batchUpdateShopPriceCase(tenantNumId, dataSign, sb, collectMap.get(integer));
                    break;
                case 504:
                    sb.append(",tiny_price =?");
                    batchUpdateShopPriceCase(tenantNumId, dataSign, sb, collectMap.get(integer));
                    break;
                case 505:
                    sb.append(",health_care_price =?");
                    batchUpdateShopPriceCase(tenantNumId, dataSign, sb, collectMap.get(integer));
                    break;
                case 506:
                    sb.append(",floor_price =?");
                    batchUpdateShopPriceCase(tenantNumId, dataSign, sb, collectMap.get(integer));
                    break;
            }
        }
    }

mdmsPProductShopPriceLogDao

java 复制代码
    public void batchUpdateShopLogNotValid(Long tenantNumId, Long dataSign, List<MDMS_P_PRODUCT_SHOP_PRICE_LOG> logList) {
        String sql = "update mdms_p_product_shop_price_log set is_valid =0,last_updtme = now(),last_update_user_id=? "
                + " where tenant_num_id = ? and data_sign = ?  and cort_num_id=? and sub_unit_num_id = ? and item_num_id = ?  "
                + " and  series=?  and cancelsign='N'";
        List<Object[]> batchArgs = new ArrayList<>();
        for (MDMS_P_PRODUCT_SHOP_PRICE_LOG log : logList) {
            Object[] object = new Object[]{tenantNumId, dataSign, 1L, log.getCORT_NUM_ID(), log.getSUB_UNIT_NUM_ID(), log.getITEM_NUM_ID(), log.getSERIES()};
            batchArgs.add(object);
        }
        int[] ints = jdbcTemplate.batchUpdate(sql, batchArgs);
        int sum = DaoUtil.sum(ints);
        if (sum != batchArgs.size()) {
            throw new DatabaseOperateException(Constant.SUB_SYSTEM, ErpExceptionType.DOE33008,
                    "批量更新零售价价格失败!");
        }
    }
相关推荐
程序无bug13 分钟前
手写Spring框架
java·后端
程序无bug15 分钟前
Spring 面向切面编程AOP 详细讲解
java·前端
全干engineer27 分钟前
Spring Boot 实现主表+明细表 Excel 导出(EasyPOI 实战)
java·spring boot·后端·excel·easypoi·excel导出
Fireworkitte36 分钟前
Java 中导出包含多个 Sheet 的 Excel 文件
java·开发语言·excel
GodKeyNet1 小时前
设计模式-责任链模式
java·设计模式·责任链模式
Paper_Love1 小时前
x86-64_windows交叉编译arm_linux程序
arm开发·windows
蓝易云1 小时前
Qt框架中connect()方法的ConnectionType参数使用说明 点击改变文章字体大小
linux·前端·后端
a_Dragon11 小时前
Spring Boot多环境开发-Profiles
java·spring boot·后端·intellij-idea
泽02021 小时前
C++之红黑树认识与实现
java·c++·rpc
ChinaRainbowSea1 小时前
补充:问题:CORS ,前后端访问跨域问题
java·spring boot·后端·spring