复制代码
<dependency>
<groupId>com.alibaba.fastjson2</groupId>
<artifactId>fastjson2</artifactId>
<version>2.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.9.0</version>
</dependency>
1. csv文件生成
java
复制代码
// 接口地址
@Value("${jk.device.uri}")
private String requestURI;
// 生成文件存放路径
@Value("${jk.file.path}")
private String filePath;
@Scheduled(cron ="0 0 1 * * ?") // 每天凌晨1点执行
public void requestDeviceData() {
// System.out.println("定时任务执行了");
DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd");
//获取前一天日期
LocalDate date = LocalDate.now().minusDays(1);
String dateStr = date.format(fmt);
String requestURL = requestURI + "?mdmTime=" + dateStr;
RestTemplate restTemplate = new RestTemplate();
JSONObject jsonObject =
restTemplate.getForObject(requestURL, JSONObject.class);
//判断有没有数据
if (jsonObject == null || jsonObject.isEmpty()) {
return;
}
JSONArray data = jsonObject.getJSONArray("data");
if (data == null || data.isEmpty()) {
return;
}
List<JkOfflineDevice> list = data.toJavaList(JkOfflineDevice.class);
if (list == null || list.isEmpty()) {
return;
}
String fileName = "mdmData_" + dateStr + ".csv";
//生成文件
generateFile(list, filePath + fileName);
}
/**
* 生成csv文件
*/
public void generateFile(List<JkOfflineDevice> list, String fileName) {
try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileName ));
CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) {
// 写入表头
csvPrinter.printRecord(
"id",
"phoneno",
"sfzh",
"imei",
"imsi",
"manufacturer",
"product",
"statusDevice",
"createTime",
"mdmTime",
"area",
"statusPhone" );
// 写入数据行
for (JkOfflineDevice jd : list) {
csvPrinter.printRecord(
jd.getId(),
jd.getPhoneno(),
jd.getSfzh(),
jd.getImei(),
jd.getImsi(),
jd.getManufacturer(),
jd.getProduct(),
jd.getStatusDevice(),
jd.getCreateTime(),
jd.getMdmTime(),
jd.getArea(),
jd.getStatusPhone()
);
}
} catch (IOException e) {
e.printStackTrace();
}
}
2. 解析csv文件
java
复制代码
@Value("${jk.ftp.path}")
private String filePath;
private static final String [] headerName = {"id","phoneno","sfzh","imei","imsi","manufacturer","product","statusDevice","createTime","mdmTime","area","statusPhone"};
public void parseFile() {
// DateTimeFormatter fmt = DateTimeFormatter.ofPattern("yyyy-MM-dd");
// //获取前一天日期
// LocalDate date = LocalDate.now().minusDays(1);
// String dateStr = date.format(fmt);
//
// String fname = filePath + "mdmData_" + dateStr + ".csv";
//
// File file = new File(fname);
//
// String fileName = file.getName();
// long time = file.lastModified();
// System.out.println("文件名：" + fileName + " 时间：" + time);
// 表记录最后更新时间， 按照这个时间读取之后的文件
// 遍历每一个文件，比较时间，大于数据库记录时间的就解析保存，小于的掠过
Integer recordId = 1;
// Long recordTime = recordMapper.getLastTimeById(recordId);
Long recordTime = 1L;
// 如果数据库记录时间不存在（说明本次操作是第一次解析同步），就使用一个比较早的时间数据
if (recordTime == null ){
recordTime = 123456789L;
}
// 遍历目录下所有文件
File directory = new File(filePath);
if (!directory.exists() || !directory.isDirectory()) {
directory.mkdirs();
}
File[] fileList = directory.listFiles();
if (fileList == null){
return;
}
for (File file : fileList){
String fileName = file.getName();
long time = file.lastModified();
// 不是csv文件，或者不是 mdmData_ 目标文件
if (!fileName.contains(".csv") || !fileName.contains("mdmData_")){
continue;
}
if (time <= recordTime){
continue;
}
List<JkOfflineDevice> list = parseCSV(file);
if (list == null || list.isEmpty()){
continue;
}
System.out.println("文件名：" + fileName + " time：" + time + "时间："+ LocalDateTime.now());
System.out.println("解析完成：" + list.size());
System.out.println(list.get(0).toString());
}
}
/**
* 把指定格式的CVS文件解析成JkOfflineDevice对象集合
* @param file
* @return
*/
private List<JkOfflineDevice> parseCSV(File file){
List<JkOfflineDevice> list = null;
if (file == null || !file.exists()){
return list;
}
try (CSVParser parser = CSVParser.parse(
new FileReader(file),
CSVFormat.DEFAULT.withFirstRecordAsHeader())) {
//表头格式校验
// id,phoneno,sfzh,imei,imsi,manufacturer,product,statusDevice,createTime,mdmTime,area,statusPhone
List<String> headerList = parser.getHeaderNames();
boolean isValid = new HashSet<>(headerList).containsAll(Arrays.asList(headerName));
//表头不是预期格式,说明该文件不是目标文件，跳过
if (!isValid) {
//System.out.println("文件格式错误，跳过");
return list;
}
// System.out.println("文件格式正确！！");
list = new ArrayList<>();
for (CSVRecord record : parser) {
JkOfflineDevice jd = new JkOfflineDevice();
jd.setId(record.get("id"));
jd.setPhoneno(record.get("phoneno"));
jd.setSfzh(record.get("sfzh"));
jd.setImei(record.get("imei"));
jd.setImsi(record.get("imsi"));
jd.setManufacturer(record.get("manufacturer"));
jd.setProduct(record.get("product"));
jd.setStatusDevice(Integer.valueOf(record.get("statusDevice")));
jd.setCreateTime(record.get("createTime"));
jd.setMdmTime(record.get("mdmTime"));
jd.setArea(record.get("area"));
jd.setStatusPhone(record.get("statusPhone"));
list.add(jd);
}
} catch (IOException e) {
list = null;
e.printStackTrace();
}
return list;
}