远程xml部分内容
css
<imagelist name="FY4A AGRI IMG REGI MTCC GLL" tag="FY4A AGRI IMG REGI MTCC GLL">
<image time="2023-07-25 22:30 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4/IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725223000_20230725223417_1000M_V0001.JPG"/>
<image time="2023-07-25 22:23 (UTC)" desc="FY4A AGRI IMG REGI MTCC GLL" url="http://img.nsmc.org.cn/PORTAL/FY4/IMG/FY4A/AGRI/IMG/REGI/MTCC/GLL/FY4A-_AGRI--_N_REGI_1047E_L1C_MTCC_MULT_GLL_20230725222336_20230725222753_1000M_V0001.JPG"/>
</imagelist>
mq发布端定时任务发送消息
css
@Component
public class TFYImage {
private ISyncFY syncFY;
Log log = LogFactory.getLog(TFYImage.class);
@Autowired
public TFYImage(ISyncFY syncFY){
this.syncFY = syncFY;
}
@PostConstruct
@Scheduled(cron = "0 0 * * * *")
public void Task1() {
log.info("execute FY Image task");
String xmlUrl = "xml path";
String localPath = "/media/resource/FY4APic/";
String type = "FY4A";
syncFY.syncFY4AImage(xmlUrl, localPath, type);
}
}
mq消费端
1,远程xml读取
2,xml解析,将image中图片url保存在集合中
3,遍历集合,当本地不存在此图片时,下载图片至本地
4,将图片路径传给延时队列,用于稍后删除图片
5,保存自定义图片访问路径等信息到数据库
css
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.databind.JsonNode;
import com.sxqx.entity.MQMessage;
import com.sxqx.listener.IMessageReceiver;
import com.sxqx.mapper.remote.xugu1.fy.IFYImageMapper;
import com.sxqx.pojo.FYImageItem;
import com.sxqx.utils.common.ListUtils;
import com.sxqx.utils.dataConverter.JsonConverter;
import com.sxqx.utils.file.FileHelper;
import com.sxqx.utils.mq.MQMessageSender;
import com.sxqx.utils.xml.XMLUtil;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.amqp.rabbit.annotation.Queue;
import org.springframework.amqp.rabbit.annotation.RabbitHandler;
import org.springframework.amqp.rabbit.annotation.RabbitListener;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import java.io.File;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
@Component
public class SyncFYImgReceiver implements IMessageReceiver {
private final IFYImageMapper fyImageMapper;
private final MQMessageSender mqMessageSender;
@Autowired
public SyncFYImgReceiver(MQMessageSender mqMessageSender,
IFYImageMapper fyImageMapper
) {
this.mqMessageSender = mqMessageSender;
this.fyImageMapper = fyImageMapper;
}
Log log = LogFactory.getLog(SyncFYImgReceiver.class);
@RabbitListener(queuesToDeclare = {
@Queue(name = "sxqxgxw_sync_fy_img")
})
@RabbitHandler
@Override
public void onMessageReceived(String mqMessageString) {
JsonNode jsonNode = JsonConverter.jsonString2JsonNode(mqMessageString);
JsonNode msg = jsonNode.findValue("msg");
JsonNode JsonNodeParams = msg.findValue("params");
Map<String, Object> params = JsonConverter.jsonNode2HashMap(JsonNodeParams);
if (params.size() > 0) {
String xmlUrl = params.get("xmlUrl").toString();
String localPath = params.get("localPath").toString();
String type = params.get("type").toString();
if(Objects.equals(type,"FY4A")){
List<String> imageUrlList = new ArrayList<>();
try {
//读取xml
Document document = XMLUtil.readXMLUrl(xmlUrl);
// 创建一个XPath对象
XPathFactory xPathFactory = XPathFactory.newInstance();
XPath xPath = xPathFactory.newXPath();
// 使用XPath表达式获取imagelist节点
XPathExpression expr = xPath.compile("/imagelist");
NodeList nodeList = (NodeList) expr.evaluate(document, XPathConstants.NODESET);
// 遍历imagelist节点
Node node = nodeList.item(0);
if (node.getNodeType() == Node.ELEMENT_NODE) {
Element imagelistElement = (Element) node;
// 获取name和tag属性值
//String name = imagelistElement.getAttribute("name");
//String tag = imagelistElement.getAttribute("tag");
//System.out.println("name: " + name);
//System.out.println("tag: " + tag);
// 获取image节点列表
NodeList imageNodes = imagelistElement.getElementsByTagName("image");
// 遍历image节点
//读取xml中image存入imageUrlList
for (int j = 0; j < imageNodes.getLength(); j++) {
Node imageNode = imageNodes.item(j);
if (imageNode.getNodeType() == Node.ELEMENT_NODE) {
Element imageElement = (Element) imageNode;
// 获取image节点的time、desc和url属性值
//String time = imageElement.getAttribute("time");
//String desc = imageElement.getAttribute("desc");
String imageUrl = imageElement.getAttribute("url");
imageUrlList.add(imageUrl);
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
List<FYImageItem> fyImageItems = new ArrayList<>();
//下载图片,组装数据
for (String imageUrl : imageUrlList) {
// 获取最后一个斜杠的位置
int slashIndex = imageUrl.lastIndexOf("/");
// 截取文件名
String fileName = imageUrl.substring(slashIndex + 1);
String filePath = localPath;
String savePath = filePath + fileName;
File file = new File(savePath);
if (file.exists()) {
continue;
}
//下载图片
try {
long start = System.currentTimeMillis();
FileHelper.downloadUsingNIO(imageUrl, savePath);
long end = System.currentTimeMillis();
long timeDifferenceInSeconds = (end - start) / 1000;
log.info(savePath + " 下载耗时:" + timeDifferenceInSeconds + "秒");
} catch (IOException e) {
e.printStackTrace();
}
if (file.exists() && file.length() > 0) {
//组装数据
String[] s = fileName.split("_");
if (s.length == 13) {
try {
String product = s[0].replace("-", "");
String dt = s[9].substring(0, 14);
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
Date date = sdf.parse(dt);
Calendar bjc = Calendar.getInstance();
bjc.setTime(date);
bjc.add(Calendar.HOUR_OF_DAY, 8);
date = bjc.getTime();
FYImageItem fyImageItem = new FYImageItem();
fyImageItem.setTime(date);
fyImageItem.setPath("http://本地服务器/mediaResource/FY4APic/" + fileName);
fyImageItem.setFilename(fileName);
fyImageItem.setProduct(product);
fyImageItems.add(fyImageItem);
} catch (Exception e) {
e.printStackTrace();
}
}
}
// 清除数据
MQMessage mqMessage = new MQMessage();
JSONObject jsonObject = new JSONObject();
jsonObject.put("filePath", savePath);
mqMessage.setMsg(jsonObject);
mqMessageSender.send("queue.file_delay_destroy", mqMessage);
}
//保存到数据库
if (fyImageItems.size() > 0) {
for (List<FYImageItem> fyImageItems1 : ListUtils.splitList(fyImageItems, 50)) {
int rows = fyImageMapper.insertIntoFYImage(fyImageItems1);
try {
Thread.sleep(50);
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
try {
Thread.sleep(100);
} catch (Exception e) {
}
} else {
log.info("don't need to sync fy4a file and fyImageItems.size() is " + fyImageItems.size());
}
}
}
}
}
XMLUtil读取远程xml链接
css
import org.w3c.dom.Document;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import java.net.HttpURLConnection;
import java.net.URL;
public class XMLUtil {
public static Document readXMLUrl(String xmlUrl) throws Exception{
// 创建一个URL对象
URL url = new URL(xmlUrl);
// 创建一个DocumentBuilderFactory对象
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
// 创建一个DocumentBuilder对象
DocumentBuilder builder = factory.newDocumentBuilder();
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
// 设置连接超时时间为5秒
connection.setConnectTimeout(5000);
// 设置读取超时时间为10秒
connection.setReadTimeout(10000);
// 从URL中读取XML文件并解析
Document document = builder.parse(connection.getInputStream());
return document;
}
}
使用NIO下载文件
css
package com.sxqx.utils.file;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
public class FileHelper {
public static void deleteFile(File file) {
/*
File[] listFiles()
返回一个抽象路径名数组,这些路径名表示此抽象路径名表示的目录中的文件。
*/
File[] files = file.listFiles();
if (files!=null) {//如果包含文件进行删除操作
for (File value : files) {
if (value.isFile()) {
//删除子文件
value.delete();
} else if (value.isDirectory()) {
//通过递归的方法找到子目录的文件
deleteFile(value);
}
value.delete();//删除子目录
}
}
file.delete();
}
public static Boolean downloadFile(String urlString, String savePath) {
InputStream is = null;
FileOutputStream os = null;
try {
// 构造URL
URL url = new URL(urlString);
// 打开连接
URLConnection con = url.openConnection();
// 输入流
is = con.getInputStream();
// 1K的数据缓冲
byte[] bs = new byte[1024];
// 读取到的数据长度
int len;
// 输出的文件流
File file = new File(savePath);
os = new FileOutputStream(file, true);
// 开始读取
while ((len = is.read(bs)) != -1) {
os.write(bs, 0, len);
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
} finally {
// 完毕,关闭所有链接
try {
if (null != os) {
os.close();
}
} catch (IOException e) {
e.printStackTrace();
}
try {
if (null != is) {
is.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
//从微信下载图片时如果没有id对应的图片则下载一个空图片,不会存在返回为null的情况
}
/**
* 使用NIO下载文件
* @param urlStr
* @param file
* @throws IOException
*/
public static void downloadUsingNIO(String urlStr, String file) throws IOException {
URL url = new URL(urlStr);
ReadableByteChannel rbc = Channels.newChannel(url.openStream());
FileOutputStream fos = new FileOutputStream(file);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}