前提:
今天接手了前同事留下的一个"屎山",一个项目中的某个接口,请求响应居然要整整5分钟,经过我的一系列优化,最终这个接口在10秒内得到了响应,并且极大的减缓了数据库和内存的压力。下面是我的调优过程
话不多说直接上代码
注意:下面是旧代码
主方法:
java
public SimScenario addSimScenarioIn2(SimScenario simScenario) throws Exception {
//把当前所有正在模拟的场景的状态更新为停止
simScenarioMapper.updateAllSimScenarioStatus("5");
simScenario.setStatus(ScenarioStatus.START);
//取得船舶数量
Scenario scenario = simScenarioService.queryScemaro(simScenario.getScenarioId());
if (simScenario.getShipNum() == null || simScenario.getShipNum() <= 0) {
simScenario.setShipNum(scenario.getShipNum());
}
simScenario.setId(UUIDGenerator.randomUUID());
simScenario.setCreateTime(new Date());
simScenario.setCreater("simulator");
simScenario.setUpdateTime(new Date());
//获取当前时间的年月日"YYYY-MM-DD"
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
String date = sdf.format(new Date());
//获取开始时间和结束时间的时分秒"HH:mm:ss"
SimpleDateFormat sdf2 = new SimpleDateFormat("HH:mm:ss");
SimpleDateFormat sdf3 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String startTimeStr = sdf2.format(scenario.getStartTime());
String endTimeStr = sdf2.format(scenario.getEndTime());
//将当前时间的年月日和开始时间的时分秒拼接成新的时间
Date startTime = sdf3.parse(date + " " + startTimeStr);
//将当前时间的年月日和结束时间的时分秒拼接成新的时间
Date endTime = sdf3.parse(date + " " + endTimeStr);
simScenario.setStartTime(startTime);
simScenario.setEndTime(endTime);
simScenario.setCreaterName("simulator");
simScenario.setScenarioName(scenario.getName());
simScenarioService.add(simScenario);
// 在船舶基础空中选取绑定的场景库的船舶数量,插入到场景船舶关系表中
List<SimScenarioShipes> simScenarioShipes = simScenarioShipesService.addShipToSimScenario(simScenario.getId(), simScenario.getShipNum());
//将插入到船舶关系表中的船插入到baseShip中
if (ObjectUtils.isNotEmpty(simScenarioShipes)) {
List<String> shipIds = simScenarioShipes.stream().map(SimScenarioShipes::getShipId).collect(Collectors.toList());
//用shipIds去t_track_ship_base表中查询mmsi集合
List<String> mmsiList = simScenarioShipesService.queryMmsiListByShipIds(shipIds);
simScenarioShipesService.addShipToBaseShipNew(simScenario.getId(), mmsiList);
}
//处理船舶轨迹数据
List<TrackScenarioShip> trackScenarioShipList = simScenarioService.queryScenarioAisByScenarioId(simScenario.getScenarioId());
List<MonitorShipAis> monitorShipAisList = new ArrayList<>();
if (ObjectUtils.isNotEmpty(trackScenarioShipList) && trackScenarioShipList.size() > 0) {
//开一个线程生成风险
new Thread(() -> {
try {
monitorWarningService.createWarningBySimScenario(simScenario.getId(), trackScenarioShipList);
} catch (ParseException e) {
e.printStackTrace();
}
}).start();
//把trackScenarioShipList数据处理成MonitorShipAis的数据并分批入库
for (TrackScenarioShip trackScenarioShip : trackScenarioShipList) {
MonitorShipAis monitorShipAis = new MonitorShipAis();
monitorShipAis.setId(UUIDGenerator.randomUUID());
monitorShipAis.setSimScenarioId(simScenario.getId());
monitorShipAis.setShipName(trackScenarioShip.getShipName());
monitorShipAis.setShipCode(trackScenarioShip.getShipCode());
monitorShipAis.setMmsi(trackScenarioShip.getMmsi());
monitorShipAis.setImo(trackScenarioShip.getImo());
//获取当前时间的年月日和gpsTime的时分秒拼接成新的时间
Date gpsTime = sdf3.parse(date + " " + sdf2.format(trackScenarioShip.getGpsTime()));
monitorShipAis.setGpsTime(gpsTime);
monitorShipAis.setLongitude(trackScenarioShip.getLongitude());
monitorShipAis.setLatitude(trackScenarioShip.getLatitude());
monitorShipAis.setDirection(trackScenarioShip.getDirection());
monitorShipAis.setShipHeadDirection(trackScenarioShip.getShipHeadDirection());
monitorShipAis.setShipLen(trackScenarioShip.getShipLen());
monitorShipAis.setShipWidth(trackScenarioShip.getShipWidth());
monitorShipAis.setCargoType(trackScenarioShip.getCargoType());
monitorShipAis.setMiles(trackScenarioShip.getMiles());
monitorShipAis.setPlanArriveTime(trackScenarioShip.getPlanArriveTime());
monitorShipAis.setShipDraft(trackScenarioShip.getShipDraft());
monitorShipAis.setCallSign(trackScenarioShip.getCallSign());
monitorShipAis.setTargetPort(trackScenarioShip.getDestPort());
monitorShipAis.setSpeed(trackScenarioShip.getSpeed());
monitorShipAisList.add(monitorShipAis);
if (monitorShipAisList.size() == 500) {
monitorShipAisService.addOrEdit(monitorShipAisList);
monitorShipAisList.clear();
}
}
if (monitorShipAisList.size() > 0) {
monitorShipAisService.addOrEdit(monitorShipAisList);
}
}
//处理事故
List<MonitorAccident> monitorAccidentList = monitorAccidentService.queryScenarioAccident(simScenario.getScenarioId());
if (ObjectUtils.isNotEmpty(monitorAccidentList) && monitorAccidentList.size() > 0) {
for (MonitorAccident accident : monitorAccidentList) {
accident.setId(UUIDGenerator.randomUUID());
accident.setSimScenarioId(simScenario.getId());
accident.setSimScenarioName(simScenario.getName());
accident.setGeom("POINT(" + accident.getLon() + " " + accident.getLat() + ")");
if (ObjectUtils.isNotEmpty(accident.getHappenTime())) {
//吧事故的时间设置为当前年月日,时分秒用happentime的时分秒拼接成新的时间
Date happenTime = sdf3.parse(date + " " + sdf2.format(accident.getHappenTime()));
accident.setHappenTime(happenTime);
}
monitorAccidentService.add(accident);
}
}
return simScenario;
}
monitorWarningService.createWarningBySimScenario()方法
java
@Override
public void createWarningBySimScenario(String simScenarioId, List<TrackScenarioShip> trackScenarioShipList) throws ParseException {
SimScenario simscenario = simScenarioService.query(simScenarioId);
//把 trackScenarioShipList中的经度,纬度 组成一个Point对象 然后组成coordinates
List<Point> coordinates = trackScenarioShipList.stream().map(ship -> {
Point point = new Point();
point.setLatitude(ship.getLatitude());
point.setLongitude(ship.getLongitude());
return point;
}).collect(Collectors.toList());
Random random = new Random();
//获取当前时间的年月日"YYYY-MM-DD"
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date();
String dateStr = sdf.format(date);
//获取开始时间和结束时间的时分秒"HH:mm:ss"
SimpleDateFormat sdf2 = new SimpleDateFormat("HH:mm:ss");
SimpleDateFormat sdf3 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
List<MonitorWarning> monitorWarnings = new ArrayList<>();
List<MonitorWarning> monitorWarningall = new ArrayList<>();
for (TrackScenarioShip trackScenarioShip : trackScenarioShipList) {
for (int i = 1; i < 7; i++) {
MonitorWarning warning = new MonitorWarning();
warning.setId(UUIDGenerator.randomUUID());
warning.setIsDelete(false);
warning.setCreater("simulator");
warning.setCreateTime(date);
warning.setUpdateTime(date);
warning.setRiskShipName(trackScenarioShip.getShipName());
warning.setRiskShipCode(trackScenarioShip.getShipCode());
warning.setHappenTime(trackScenarioShip.getGpsTime());
warning.setSimScenarioId(simscenario.getId());
warning.setWaterAreaId(simscenario.getWaterAreaId());
warning.setPredictionTime(trackScenarioShip.getGpsTime());
warning.setLon(trackScenarioShip.getLongitude().toString());
warning.setLat(trackScenarioShip.getLatitude().toString());
warning.setRiskShipCode(trackScenarioShip.getMmsi());
warning.setSimScenarioName(simscenario.getName());
warning.setGeom("POINT(" + warning.getLon() + " " + warning.getLat() + ")");
if (ObjectUtils.isNotEmpty(warning.getHappenTime())) {
//吧风险的时间设置为当前年月日,时分秒用happentime的时分秒拼接成新的时间
Date happenTime = sdf3.parse(dateStr + " " + sdf2.format(warning.getHappenTime()));
warning.setHappenTime(happenTime);
}
Map<String, String> map = new LinkedHashMap<>();
map.put("01", "碰撞");
map.put("02", "搁浅");
map.put("03", "触碰");
map.put("04", "翻沉");
map.put("05", "偏航");
map.put("06", "火灾");
String s = map.get("0" + i);
warning.setRiskType("0" + i);
warning.setRiskName(s);
Date selectTime = warning.getHappenTime();
//筛选trackScenarioShipList时间在selectTime前一分钟和后一分钟的数据 并生成点集合
List<Point> collect = trackScenarioShipList.stream().filter(ship -> {
long time = ship.getGpsTime().getTime();
long selectTime1 = selectTime.getTime();
return time >= selectTime1 - 60000 && time <= selectTime1 + 60000;
}).map(ship -> {
Point point = new Point();
point.setLatitude(ship.getLatitude());
point.setLongitude(ship.getLongitude());
return point;
}).collect(Collectors.toList());
handleRiskWarning(warning,collect);
monitorWarnings.add(warning);
monitorWarningall.add(warning);
}
//如果monitorWarnings的size大于500 就分批插入
if (monitorWarnings.size() > 500) {
int size = monitorWarnings.size();
int batchSize = 500;
for (int i = 0; i < size; i += batchSize) {
int end = Math.min(i + batchSize, size);
List<MonitorWarning> monitorWarningsasesBatch = monitorWarnings.subList(i, end);
monitorWarningMapper.insertBatch(monitorWarningsasesBatch);
}
monitorWarnings.clear();
}
}
//插入剩余的数据
if (monitorWarnings.size() > 0) {
monitorWarningMapper.insertBatch(monitorWarnings);
}
Map<String, List<Object>> toMap = monitorWarningall.stream().collect(
Collectors.groupingBy(
MonitorWarning::getRiskShipCode,
Collectors.collectingAndThen(
Collectors.toList(),
list -> {
list.sort(Comparator.comparingLong(m -> ((MonitorWarning) m).getHappenTime().getTime()));
return new ArrayList<Object>(list);
}
)
)
);
Map<String, JSONArray> map = toMap.entrySet().stream().collect(Collectors.toMap(m -> String.valueOf("warning_" + m.getKey()), m -> new JSONArray(m.getValue())));
// 清除redis db 模拟场景数据
Set<String> keys = redisTemplate.keys("warning_*");
Long deleteCount = redisTemplate.delete(keys);
log.info("-----------> 已清除" + deleteCount + " 条缓存数据");
// 缓存数据
redisTemplate.opsForValue().multiSet(map);
log.info("-----------> 已缓存" + map.size() + " 条缓存数据");
}
private void handleRiskWarning(MonitorWarning warning,List<Point> coordinates) {
if (warning.getRiskType().equals("01")) {
Point point1 = new Point(Double.parseDouble(warning.getLon()), Double.parseDouble(warning.getLat()));
Double minDistance = calculateAverageShortestDistance(point1,coordinates);
if (ObjectUtils.isEmpty(minDistance)){
warning.setRiskValue(0.01);
} else if ( minDistance < 10 ) {
warning.setRiskValue(1.0);
} else if (minDistance < 100 && minDistance >= 10) {
warning.setRiskValue((new BigDecimal(Double.toString((1.0 - (minDistance / 100.0))*0.8)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()));
} else if (minDistance < 1000 && minDistance >= 100) {
warning.setRiskValue((new BigDecimal(Double.toString((1.0 - (minDistance / 1000.0))*0.4)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()));
} else if (minDistance >= 1000) {
warning.setRiskValue(new BigDecimal(Double.toString(new Random().nextDouble() * 0.01)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue());
}
} else if (warning.getRiskType().equals("02")) {
Double minDistance = monitorWarningMapper.minDistance(warning.getLon(), warning.getLat(),warning.getWaterAreaId());
//List<Double> speed = monitorWarningMapper.getSpeed(warning.getRiskShipCode(),warning.getLon(), warning.getLat(),warning.getSimScenarioId());
if (ObjectUtils.isEmpty(minDistance)){
warning.setRiskValue(0.01);
}else if ( minDistance < 10 ) {
warning.setRiskValue(0.8);
} else if (minDistance < 100 && minDistance >= 10) {
warning.setRiskValue(Math.max(0.01, new BigDecimal(Double.toString((0.8 - (minDistance / 100.0))*0.8)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()));
} else if (minDistance < 1000 && minDistance >= 100) {
warning.setRiskValue(Math.max(0.01, new BigDecimal(Double.toString((0.8 - (minDistance / 1000.0))*0.4)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()));
} else if (minDistance >= 1000) {
warning.setRiskValue(new BigDecimal(Double.toString(new Random().nextDouble() * 0.01)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue());
}
} else if (warning.getRiskType().equals("03")) {
Double minDistance = monitorWarningMapper.minDistanceFromFacility(warning.getLon(), warning.getLat(),warning.getWaterAreaId());
if (ObjectUtils.isEmpty(minDistance)){
warning.setRiskValue(0.01);
} else if ( minDistance == 0 ) {
warning.setRiskValue(1.0);
} else if (minDistance < 500 && minDistance >= 0) {
warning.setRiskValue((new BigDecimal(Double.toString((1.0 - (minDistance / 500.0))*0.8)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()));
} else if (minDistance < 5000 && minDistance >= 500) {
warning.setRiskValue((new BigDecimal(Double.toString((1.0 - (minDistance / 5000.0))*0.4)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue()));
} else if ( minDistance >= 5000 ) {
warning.setRiskValue(0.0);
}
} else if (warning.getRiskType().equals("04")) {
//随机生成一个0.1以下的double类型的值
warning.setRiskValue(new BigDecimal(Double.toString(new Random().nextDouble() * 0.1)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue());
} else if (warning.getRiskType().equals("05")) {
//随机生成一个0.2以下的double类型的值
warning.setRiskValue(new BigDecimal(Double.toString(new Random().nextDouble() * 0.2)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue());
} else if (warning.getRiskType().equals("06")) {
//随机生成一个0.2以下的double类型的值
warning.setRiskValue(new BigDecimal(Double.toString(new Random().nextDouble() * 0.2)).setScale(2, BigDecimal.ROUND_HALF_UP).doubleValue());
}
}
// 计算任意一个点到其他点的最短距离的平均值
private double calculateAverageShortestDistance(Point point1,List<Point> coordinates) {
double minDistance = Double.POSITIVE_INFINITY;
//去除coordinates中Point1自己的经纬度
List<Point> collect = coordinates.stream().filter(point -> !point.equals(point1)).collect(Collectors.toList());
//计算Point1到其他点的最短距离的平均值
for (Point point : collect) {
double distance = this.haversineDistance(point1, point); // 计算点之间的距离
if (distance < minDistance) {
minDistance = distance; // 更新最短距离
}
}
return minDistance;
}
// 使用Haversine公式计算两个经纬度坐标点之间的距离
private double haversineDistance(Point point1, Point point2) {
double lon1 = point1.getLongitude();
double lat1 = point1.getLatitude();
double lon2 = point2.getLongitude();
double lat2 = point2.getLatitude();
// 将经纬度转换为弧度
lon1 = Math.toRadians(lon1);
lat1 = Math.toRadians(lat1);
lon2 = Math.toRadians(lon2);
lat2 = Math.toRadians(lat2);
// Haversine公式计算距离
double dlon = lon2 - lon1;
double dlat = lat2 - lat1;
double a = Math.sin(dlat / 2) * Math.sin(dlat / 2) + Math.cos(lat1) * Math.cos(lat2) * Math.sin(dlon / 2) * Math.sin(dlon / 2);
double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
double distance = 6371000 * c; // 地球半径为6371km
return distance;
}
其实从上面的代码中进行简单的分析就可以得知,这个接口响应慢的,内存占用率高的原因有以下几点
主方法内
- 开一个线程生成风险这部分的线程没有被线程池管理,容易造成资源浪费,线程没关闭等风险
- for循环处理trackScenarioShipList,这个集合数据超过20000条
- 插入数据的时候,没有采用批处理提交,而是多次提交
计算风险的方法**createWarningBySimScenario()**方法内
- createWarningBySimScenario()内的handleRiskWarning()方法。此方法居然使用数据库处理空间数据,要知道集合内有超过20000条数据啊,也就是说这个方法会循环查询20000次数据库,还是带空间计算的。
- 依旧是老问题 插入数据的时候,没有采用批处理提交,而是多次提交。
现在我们先解决主方法内的第一个问题,线程问题
主方法问题一、线程问题
首先我决定采用线程池来管理这个线程,通过springboot自带的@Async来处理
- 创建线程池配置类
java
package com.ztjz.itp.swm.sim.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.AsyncConfigurer;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.Executor;
import java.util.concurrent.ThreadPoolExecutor;
/**
* @author Felix
* @describe:
* @date 2026年01月15日 15:37
*/
@Configuration
@EnableAsync
public class AsyncConfig implements AsyncConfigurer {
// 默认异步任务执行器
@Override
public Executor getAsyncExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 核心线程数 = CPU核心数
int corePoolSize = Runtime.getRuntime().availableProcessors();
executor.setCorePoolSize(corePoolSize);
// 最大线程数 = CPU核心数 * 2
int maxPoolSize = corePoolSize * 2;
executor.setMaxPoolSize(maxPoolSize);
// 队列容量:针对大数据量处理增大队列容量
executor.setQueueCapacity(1000);
// 空闲线程存活时间
executor.setKeepAliveSeconds(60);
// 线程名前缀
executor.setThreadNamePrefix("Async-");
// 拒绝策略:调用者运行
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
// 优雅关闭
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(60);
executor.initialize();
return executor;
}
// 风险预警专用线程池
@Bean("riskWarningExecutor")
public Executor riskWarningExecutor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
// 针对大数据量处理优化配置
executor.setCorePoolSize(20);
executor.setMaxPoolSize(40);
executor.setQueueCapacity(2000);
executor.setKeepAliveSeconds(60);
executor.setThreadNamePrefix("risk-warning-");
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
executor.setWaitForTasksToCompleteOnShutdown(true);
executor.setAwaitTerminationSeconds(60);
executor.setAllowCoreThreadTimeOut(true);
executor.initialize();
return executor;
}
}
- 在目标方法上面加上注解
java
@Override
@Async("riskWarningExecutor")
public void createWarningBySimScenario(String simScenarioId, List<TrackScenarioShip> trackScenarioShipList) throws ParseException {
SimScenario simscenario = simScenarioService.query(simScenarioId);
//获取当前时间的年月日"YYYY-MM-DD"
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
Date currentTime = new Date();
String dateStr = sdf.format(currentTime);
//获取开始时间和结束时间的时分秒"HH:mm:ss"
....下方代码省略
主方法问题二、单线程for循环处理集合太慢
采用java8的stream流处理,改造后方法如下
java
// 线程安全的时间格式化器
private static final DateTimeFormatter DATE_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final DateTimeFormatter TIME_FORMATTER =
DateTimeFormatter.ofPattern("HH:mm:ss");
private static final DateTimeFormatter DATETIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
List<MonitorShipAis> monitorShipAisList = trackScenarioShipList.parallelStream().map(trackScenarioShip->{
MonitorShipAis monitorShipAis = new MonitorShipAis();
monitorShipAis.setId(UUIDGenerator.randomUUID());
monitorShipAis.setSimScenarioId(simScenario.getId());
monitorShipAis.setShipName(trackScenarioShip.getShipName());
monitorShipAis.setShipCode(trackScenarioShip.getShipCode());
monitorShipAis.setMmsi(trackScenarioShip.getMmsi());
monitorShipAis.setImo(trackScenarioShip.getImo());
// 优化时间处理 - 使用线程安全的时间格式化
ZonedDateTime currentZdt = currentTime.toInstant()
.atZone(ZoneId.of("Asia/Shanghai")); // 转换为带时区的时间
ZonedDateTime gpsZdt = trackScenarioShip.getGpsTime().toInstant()
.atZone(ZoneId.of("Asia/Shanghai"));
String dateStr = DATE_FORMATTER.format(currentZdt);
String timeStr = TIME_FORMATTER.format(gpsZdt);
String datetimeStr = dateStr + " " + timeStr;
// 解析为ZonedDateTime(带时区信息)
ZonedDateTime zdt = LocalDateTime.parse(datetimeStr, DATETIME_FORMATTER)
.atZone(ZoneId.of("Asia/Shanghai"));
Date gpsTime = Date.from(zdt.toInstant());
monitorShipAis.setGpsTime(gpsTime);
monitorShipAis.setLongitude(trackScenarioShip.getLongitude());
monitorShipAis.setLatitude(trackScenarioShip.getLatitude());
monitorShipAis.setDirection(trackScenarioShip.getDirection());
monitorShipAis.setShipHeadDirection(trackScenarioShip.getShipHeadDirection());
monitorShipAis.setShipLen(trackScenarioShip.getShipLen());
monitorShipAis.setShipWidth(trackScenarioShip.getShipWidth());
monitorShipAis.setCargoType(trackScenarioShip.getCargoType());
monitorShipAis.setMiles(trackScenarioShip.getMiles());
monitorShipAis.setPlanArriveTime(trackScenarioShip.getPlanArriveTime());
monitorShipAis.setShipDraft(trackScenarioShip.getShipDraft());
monitorShipAis.setCallSign(trackScenarioShip.getCallSign());
monitorShipAis.setTargetPort(trackScenarioShip.getDestPort());
monitorShipAis.setSpeed(trackScenarioShip.getSpeed());
return monitorShipAis;
}).collect(Collectors.toList());
其中的核心点就是把普通for循环 改为了parallelStream 并行流处理,需要注意的是SimpleDateFormat 这个时间格式处理器不是线程安全的,在多线程中可能存在问题,所以我们把时间格式处理改为线程安全的DateTimeFormatter。
主方法问题三、批量插入数据
- 修改数据库连接配置,我们采用的是postgre数据库,所以需要在连接数据库时开启批处理功能
数据库原本连接信息:
jdbc:postgresql://localhost:5236/swmdb
修改后:
jdbc:postgresql://localhost:5236/swmdb?reWriteBatchedInserts=true&defaultRowFetchSize=1000 - 配置jpa批处理()
yaml
spring:
jpa:
properties:
hibernate:
jdbc:
batch_size: 50
batch_versioned_data: true
order_inserts: true
order_updates: true
- 最关键的一点
java
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
@Resource
private SqlSessionFactory sqlSessionFactory;
@Override
public List<MonitorShipAis> addBatch(List<MonitorShipAis> monitorShipAisList) {
int batchSize = 1000; // 减小批次大小
int total = monitorShipAisList.size();
try (SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false)) {
MonitorShipAisMapper mapper = sqlSession.getMapper(MonitorShipAisMapper.class);
for (int i = 0; i < total; i++) {
MonitorShipAis monitorShipAis = monitorShipAisList.get(i);
mapper.insert(monitorShipAis);
// 每批次刷新
if (i > 0 && i % batchSize == 0) {
sqlSession.flushStatements();
}
}
// 刷新剩余
sqlSession.flushStatements();
sqlSession.commit();
} catch (Exception e) {
log.error("批量插入失败,数据量: {}", total, e);
throw e;
}
return monitorShipAisList;
}
代码解释说明:
1. sqlSession.flushStatements() 作用:
执行批量语句: 将当前累积在内存中的批量 SQL 语句(insert/update/delete)一次性发送到数据库执行
清空语句缓冲区: 清空 JDBC 的 PreparedStatement 缓存
不提交事务: 仅执行但不提交,数据库事务仍然保持打开状态
2. sqlSession.commit() 作用:
提交事务: 将当前事务中的所有操作永久保存到数据库
释放资源: 完成事务处理
结束事务: 事务提交后不能再回滚
上面批处理代码,每次发送1000条数据到数据库中执行,最后把内存的全部数据都发送到数据库中执行以后,提交事务,释放数据库连接。
计算风险方法问题、空间函数处理
这里我们采用 JTS (Java Topology Suite) 库来实现空间函数运算,原因如下
性能提升: 避免了每条数据都发起数据库请求(Round-trip)的网络开销,在处理 20000 条轨迹数据时能显著提速。
解耦计算: 将计算逻辑下沉到业务代码中,不依赖数据库的存储过程,更易于维护和单元测试。
- 引入依赖
xml
<!-- JTS (Java Topology Suite) -->
<dependency>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>1.19.0</version>
</dependency>
<dependency>
<groupId>org.locationtech.jts.io</groupId>
<artifactId>jts-io-common</artifactId>
<version>1.19.0</version>
</dependency>
- 创建空间计算工具类
java
package com.ztjz.itp.swm.sim.util;
import org.locationtech.jts.geom.*;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKTReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Felix
* @describe:
* @date 2026年01月15日 16:55
*/
public class GeometryUtils {
private static final Logger logger = LoggerFactory.getLogger(GeometryUtils.class);
private static final GeometryFactory geometryFactory = new GeometryFactory();
private static final WKTReader wktReader = new WKTReader(geometryFactory);
/**
* 创建点几何对象
*/
public static Point createPoint(double x, double y) {
return geometryFactory.createPoint(new Coordinate(x, y));
}
/**
* 创建多边形几何对象
*/
public static Polygon createPolygon(double[][] coordinates) {
Coordinate[] coords = new Coordinate[coordinates.length + 1];
for (int i = 0; i < coordinates.length; i++) {
coords[i] = new Coordinate(coordinates[i][0], coordinates[i][1]);
}
coords[coordinates.length] = coords[0]; // 闭合多边形
return geometryFactory.createPolygon(coords);
}
/**
* 计算两个几何对象之间的距离
*/
public static double calculateDistance(Geometry geom1, Geometry geom2) {
return geom1.distance(geom2);
}
/**
* 获取几何对象的边界
*/
public static Geometry getBoundary(Geometry geometry) {
return geometry.getBoundary();
}
/**
* 解析WKT字符串为几何对象
*/
public static Geometry parseWKT(String wkt) {
try {
return wktReader.read(wkt);
} catch (ParseException e) {
logger.error("Failed to parse WKT: " + wkt, e);
throw new RuntimeException("Invalid geometry WKT", e);
}
}
/**
* 将几何对象转换为WKT字符串
*/
public static String toWKT(Geometry geometry) {
return geometry.toText();
}
/**
* 检查几何对象是否有效
*/
public static boolean isValid(Geometry geometry) {
return geometry != null && geometry.isValid();
}
/**
* 计算点到多边形边界的最小距离
*/
public static double distanceToPolygonBoundary(double x, double y, Polygon polygon) {
Point point = createPoint(x, y);
Geometry boundary = getBoundary(polygon);
return calculateDistance(point, boundary);
}
/**
* 将度数转换为米(近似值)
*/
public static double degreesToMeters(double degrees) {
return degrees * 111320; // 1度约等于111320米
}
}
-
代码修改
把handleRiskWarning()方法中的这行代码
Double minDistance = monitorWarningMapper.minDistance(warning.getLon(), warning.getLat(),warning.getWaterAreaId());修改为
Double minDistance = this.calculateMinDistance(warning.getLon(), warning.getLat(),warning.getWaterAreaId());calculateMinDistance()方法如下
java
public double calculateMinDistance(String lonStr, String latStr, String waterAreaId) {
Double lon = Double.valueOf(lonStr);
Double lat = Double.valueOf(latStr);
// 创建点几何对象
org.locationtech.jts.geom.Point point = GeometryUtils.createPoint(lon, lat);
// 从数据库获取水域几何数据
String geometryWkt = areaWktCache.get(waterAreaId);
if(StringUtils.isBlank(geometryWkt)){
geometryWkt = monitorWarningMapper.getGeometryWkt(waterAreaId);
areaWktCache.put(waterAreaId,geometryWkt);
}
// 解析几何对象
Geometry waterGeometry = GeometryUtils.parseWKT(geometryWkt);
// 获取几何边界
Geometry boundary = waterGeometry.getBoundary();
// 计算距离并转换为米
return point.distance(boundary) * 111320;
}
至此,就完成了整个接口的优化
总结:
1、避免for循环插入数据库,计算代码尽量放到代码中计算,而不是数据库中计算
2、大批量数据库入库时,尽量只在最后提交一次事务,避免反复的连接数据库