java流式处理zip+多线程

概述

流式处理一个zip,zip里有多个json文件。

流式处理可以避免解压一个大的zip。再加上多线程,处理的效率杠杠的。

代码

java 复制代码
package 多线程.demo05多jsonCountDownLatch;

import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StopWatch;

import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

@Slf4j
public class ZipProcessor {

    private Path path;

    private static int numThreads = Runtime.getRuntime().availableProcessors();
    private static ExecutorService executorService = Executors.newFixedThreadPool(numThreads);
    private static ObjectMapper objectMapper = new ObjectMapper();

    @SneakyThrows
    public ZipProcessor(String filePath){
        path = Paths.get(filePath);
        if (!Files.exists(path)) {
            throw new FileNotFoundException("The specified ZIP file does not exist: " + filePath);
        }
    }

    public void streamProcess(){
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        // 使用 try-with-resources 保证资源关闭
        try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(path))) {
            ZipEntry entry;
            while ((entry = zis.getNextEntry()) != null) {
                // 将当前条目的数据读取到字节数组中
                byte[] byteArray = getByteArray(zis);
                process(byteArray, entry);
                // 关闭当前条目的输入流
                zis.closeEntry();
            }
            stopWatch.stop();
            log.info("zip处理耗时:{}秒", stopWatch.getTotalTimeSeconds());
        } catch (IOException e) {
            stopWatch.stop();
            log.error("zip处理异常,耗时:{}秒", stopWatch.getTotalTimeSeconds(), e);
        }

    }

    public void streamParallelProcess() {
        StopWatch stopWatch = new StopWatch();
        stopWatch.start();
        try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(path))) {
            ZipEntry entry;
            List<Future<?>> futures = new ArrayList<>();
            while ((entry = zis.getNextEntry()) != null) {
                // 为了lambda表达式捕获局部变量
                final ZipEntry currentEntry = entry;
                // 将当前条目的数据读取到字节数组中
                byte[] byteArray = getByteArray(zis);
                Future<?> future = executorService.submit(() -> process(byteArray, currentEntry));
                futures.add(future);
            }

            // 等待所有任务完成
            for (Future<?> future : futures) {
                try {
                    future.get();
                } catch (Exception e) {
                    log.error("任务执行失败", e);
                }
            }
        } catch (IOException e) {
            log.error("读取ZIP文件异常", e);
        } finally {
            executorService.shutdown();
            try {
                if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) {
                    executorService.shutdownNow();
                }
            } catch (InterruptedException ex) {
                executorService.shutdownNow();
            }
            stopWatch.stop();
            log.info("zip处理耗时:{}秒", stopWatch.getTotalTimeSeconds());
        }
    }

    private void process(byte[] entryData, ZipEntry entry) {
        try {
            // 在这里处理每个条目的数据
            ObjectMapper objectMapper = new ObjectMapper();
            OriginalObject originalObject = objectMapper.readValue(entryData, OriginalObject.class);
            log.info("完成处理:{},sourceFileId:{}", entry.getName(), originalObject.getSourceFileId());
        } catch (IOException e) {
            log.error("处理条目 {} 异常", entry.getName(), e);
        }
    }

    private byte[] getByteArray(ZipInputStream zis) throws IOException {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        byte[] buffer = new byte[1024];
        int length;
        while ((length = zis.read(buffer)) > 0) {
            baos.write(buffer, 0, length);
        }
        return baos.toByteArray();
    }

}
相关推荐
2401_897930063 分钟前
tensorflow常用使用场景
人工智能·python·tensorflow
酷飞飞2 小时前
错误是ModuleNotFoundError: No module named ‘pip‘解决“找不到 pip”
人工智能·python·pip
点云SLAM3 小时前
PyTorch 中.backward() 详解使用
人工智能·pytorch·python·深度学习·算法·机器学习·机器人
B1118521Y463 小时前
flask的使用
后端·python·flask
only-qi3 小时前
146. LRU 缓存
java·算法·缓存
Learn Beyond Limits4 小时前
Transfer Learning|迁移学习
人工智能·python·深度学习·神经网络·机器学习·ai·吴恩达
xuxie135 小时前
SpringBoot文件下载(多文件以zip形式,单文件格式不变)
java·spring boot·后端
重生成为编程大王5 小时前
Java中的多态有什么用?
java·后端
666和7775 小时前
Struts2 工作总结
java·数据库
中草药z5 小时前
【Stream API】高效简化集合处理
java·前端·javascript·stream·parallelstream·并行流