文件上传分片上传

分片上传则是将一个大文件分割成多个小块分别上传，最后再由服务器合并成完整的文件。这种做法的好处是可以并行处理多个小文件，提高上传效率；同时，如果某一部分上传失败，只需要重传这一部分，不影响其他部分。

初步实现

后端代码

java 复制代码

/**
 * 分片上传
 *
 * @param file 上传的文件
 * @param start 文件开始上传的位置
 * @param fileName 文件名称
 * @return  上传结果
 */
@PostMapping("/fragmentUpload")
@ResponseBody
public AjaxResult fragmentUpload(@RequestParam("file") MultipartFile file, @RequestParam("start") long start, @RequestParam("fileName") String fileName) {
    try {
        // 检查上传目录是否存在，如果不存在则创建
        File directory = new File(uploadPath);
        if (!directory.exists()) {
            directory.mkdirs();
        }

        // 设置上传文件的目标路径
        File targetFile = new File(uploadPath +File.separator+ fileName);
        // 创建 RandomAccessFile 对象以便进行文件的随机读写操作
        RandomAccessFile randomAccessFile = new RandomAccessFile(targetFile, "rw");
        // 获取 RandomAccessFile 对应的 FileChannel
        FileChannel channel = randomAccessFile.getChannel();
        // 设置文件通道的位置，即从哪里开始写入文件内容
        channel.position(start);
        // 从 MultipartFile 对象的资源通道中读取文件内容，并写入到指定位置
        channel.transferFrom(file.getResource().readableChannel(), start, file.getSize());

        // 关闭文件通道和 RandomAccessFile 对象
        channel.close();
        randomAccessFile.close();

        // 返回上传成功的响应
        return AjaxResult.success("上传成功");
    } catch (Exception e) {
        // 捕获异常并返回上传失败的响应
        return AjaxResult.error("上传失败");
    }
}

/**
 * 检测文件是否存在
 * 如果文件存在，则返回已经存在的文件大小。
 * 如果文件不存在，则返回 0，表示前端从头开始上传该文件。
 * @param filename
 * @return
 */
@GetMapping("/checkFile")
@ResponseBody
public AjaxResult checkFile(@RequestParam("filename") String filename) {
    File file = new File(uploadPath+File.separator + filename);
    if (file.exists()) {
        return AjaxResult.success(file.length());
    } else {
        return AjaxResult.success(0L);
    }
}

前端

cpp 复制代码

var prefix = ctx + "/kuroshiro/file-upload";

// 每次上传大小
const chunkSize = 1 * 1024 * 1024;

/**
 * 开始上传
 */
function startUpload(type) {
    const fileInput = document.getElementById('fileInput');
    const file = fileInput.files[0];
    if (!file) {
        alert("请选择文件");
        return;
    }
    if(type == 1){
	    checkFile(filename).then(start => {
	         uploadFile(file, start,Math.min(start + chunkSize, file.size));
	     })
	 }
}

/**
 * 检查是否上传过
 * @param filename
 * @returns {Promise<unknown>}
 */
function checkFile(filename) {
    return $fetch(prefix+`/checkFile?filename=${filename}`);
}

/**
 * 开始分片上传
 * @param file 文件
 * @param start 开始位置
 * @param end 结束位置
 */
function uploadFile(file, start,end) {
    if(start < end){
        const chunk = file.slice(start, end);
        const formData = new FormData();
        formData.append('file', chunk);
        formData.append('start', start);
        formData.append('fileName', file.name);

        $fetch(prefix+'/fragmentUpload', {
            method: 'POST',
            body: formData
        }).then(response => {
            console.log(`分片 ${start} - ${end} 上传成功`);
            // 递归调用
            uploadFile(file,end,Math.min(end + chunkSize, file.size))
        })
    }

}

function $fetch(url,requestInit){
        return new Promise((resolve, reject) => {
            fetch(url,requestInit).then(response => {
                if (!response.ok) {
                    throw new Error('请求失败');
                }
                return response.json();
            }).then(data => {
            if (data.code === 0) {
                resolve(data.data);
            } else {
                console.error(data.msg);
                reject(data.msg)
            }
            }).catch(error => {
                console.error(error);
                reject(error)
            });
        });
    }

以上虽然实现的分片上传，但是它是某种意义上来说还是与整体上传差不多，它是一段一段的上传，某段上传失败后，后续的就不会再继续上传；不过比起整体上传来说，它会保存之前上传的内容，下一个上传时，从之前上传的位置接着上传。不用整体上传。下面进行优化。

优化

首先，之前的分片上传，后端是直接写入了一个文件中了，所以只能顺序的上传写入，虽然可以保存上传出错之前的内容，但是整体上看来是速度也不行。

优化逻辑：把分片按顺序单独保存下来，等到所有分片都上传成功后，把所有分片合并成文件。这样上传的时候就不用等着上一个上传成功才上传下一个了。

后端代码

cpp 复制代码

/**
* 分片上传
 * @param file 文件
 * @param chunkIndex 分片下标
 */
@PostMapping("/uploadChunk")
@ResponseBody
public AjaxResult uploadChunk(@RequestParam("file") MultipartFile file, @RequestParam("chunkIndex") int chunkIndex,@RequestParam("fileName") String fileName) {

    String uploadDirectory = chunkUploadPath+File.separator+fileName;
    File directory = new File(uploadDirectory);
    if (!directory.exists()||directory.isFile()) {
        directory.mkdirs();
    }
    String filePath = uploadDirectory + File.separator + fileName+ "_" + chunkIndex;
    try (OutputStream os = new FileOutputStream(filePath)) {
        os.write(file.getBytes());
        return AjaxResult.success("分片"+(chunkIndex+1)+"上传成功");
    }catch (Exception e){
        // 保存失败后如果文件建立了就删除，下次上传时重新保存，避免文件内容错误
        File chunkFile = new File(filePath);
        if(chunkFile.exists()) chunkFile.delete();
        e.printStackTrace();
        return AjaxResult.error("分片"+(chunkIndex+1)+"上传失败");
    }

}


/**
 * 检测分片是否存在
 * 如果文件存在，则返回已经存在的分片下标集合。存在的就不上传
 * 如果文件不存在，则返回空集合，表示前端从头开始上传该文件
 * @param fileName
 * @return
 */
@GetMapping("/checkChunk")
@ResponseBody
public AjaxResult checkChunk(@RequestParam("fileName") String fileName) {
    String uploadDirectory = chunkUploadPath+File.separator+fileName;
    List<Integer> list = new ArrayList<>();
    File file = new File(uploadDirectory);
    // 文件目录不存在
    if(!file.exists()||file.isFile()) return AjaxResult.success(list);

    File[] files = file.listFiles();
    // 文件目录下没有分片文件
    if(files == null) return AjaxResult.success(list);

    // 返回存在分片下标集合
    return AjaxResult.success(Arrays.stream(files).map(item->Integer.valueOf(item.getName().substring(item.getName().lastIndexOf("_")+1))).collect(Collectors.toList()));
}

    // 合并文件分片
    @PostMapping("/mergeChunks")
    @ResponseBody
    public AjaxResult mergeChunks(@RequestParam("fileName") String fileName, @RequestParam("totalChunks") int totalChunks) {
        String uploadDirectory = chunkUploadPath+File.separator+fileName;
        String mergedFilePath = uploadPath +File.separator+ fileName;

        try (OutputStream os = new FileOutputStream(mergedFilePath, true)) {
            for (int i = 0; i < totalChunks; i++) {
                Path chunkFilePath = Paths.get(uploadDirectory +File.separator+ fileName + "_" + i);
                Files.copy(chunkFilePath, os);
                Files.delete(chunkFilePath);
            }
            return AjaxResult.success();
        }catch (Exception e){
            e.printStackTrace();
            return AjaxResult.error(e.getMessage());
        }

    }

前端代码

cpp 复制代码

/**
 * 开始上传
 */
function startUpload(type) {
    const fileInput = document.getElementById('fileInput');
    const file = fileInput.files[0];
    if (!file) {
        alert("请选择文件");
        return;
    }

    
     const filename = file.name;
     if(type == 1){
         checkFile(filename).then(start => {
             uploadFile(file, start,Math.min(start + chunkSize, file.size));
         })
     }
     if(type == 2){
         checkChunk(filename).then(arr => {
             uploadChunk(file, arr);
         })
     }
}

 /**
* 切割文件为多个分片
* @param file
* @returns {*[]}
*/
function sliceFile(file) {
   const chunks = [];
   let offset = 0;

   while (offset < file.size) {
       const chunk = file.slice(offset, offset + chunkSize);
       chunks.push(chunk);
       offset += chunkSize;
   }

   return chunks;
}
/**
* 检查是否上传过
* @param filename
* @returns {Promise<unknown>}
*/
function checkChunk(filename) {
   return $fetch(prefix+`/checkChunk?fileName=${filename}`);
}

/**
* 开始分片上传
* @param file 文件
* @param exists 存在的分片下标
*/
function uploadChunk(file,exists) {
   const chunkArr = sliceFile(file);
   Promise.all(chunkArr.map((chunk, index) => {
       if(!exists.includes(index)){
           const formData = new FormData();
           formData.append('file', chunk);
           formData.append('fileName', file.name);
           formData.append('chunkIndex', index);

           return $fetch(prefix+'/uploadChunk', {
               method: 'POST',
               body: formData
           });
       }
   })).then(uploadRes=> {
       // 合并分片
       const formData = new FormData();
       formData.append('fileName', file.name);
       formData.append('totalChunks', chunkArr.length);
       $fetch(prefix + '/mergeChunks', {
           method: 'POST',
           body:formData,
       }).then(mergeRes=>{
           console.log("合并成功")
       });
   });
}

以上优化后所有分片可以同时上传，所有分片上传都成功后进行合并。

最后是完整代码

cpp 复制代码

@Controller()
@RequestMapping("/kuroshiro/file-upload")
public class FileUploadController {
    private String prefix = "kuroshiro/fragmentUpload";
    // 文件保存目录
    private final String uploadPath = RuoYiConfig.getUploadPath();
    // 分片保存目录
    private final String chunkUploadPath = uploadPath+File.separator+"chunks";

    /**
     * demo
     * @return
     */
    @GetMapping("/demo")
    public String demo() {
        return prefix+"/demo";
    }

    /**
     * 分片上传
     *
     * @param file 上传的文件
     * @param start 文件开始上传的位置
     * @param fileName 文件名称
     * @return  上传结果
     */
    @PostMapping("/fragmentUpload")
    @ResponseBody
    public AjaxResult fragmentUpload(@RequestParam("file") MultipartFile file, @RequestParam("start") long start, @RequestParam("fileName") String fileName) {
        try {
            // 检查上传目录是否存在，如果不存在则创建
            File directory = new File(uploadPath);
            if (!directory.exists()) {
                directory.mkdirs();
            }

            // 设置上传文件的目标路径
            File targetFile = new File(uploadPath +File.separator+ fileName);
            // 创建 RandomAccessFile 对象以便进行文件的随机读写操作
            RandomAccessFile randomAccessFile = new RandomAccessFile(targetFile, "rw");
            // 获取 RandomAccessFile 对应的 FileChannel
            FileChannel channel = randomAccessFile.getChannel();
            // 设置文件通道的位置，即从哪里开始写入文件内容
            channel.position(start);
            // 从 MultipartFile 对象的资源通道中读取文件内容，并写入到指定位置
            channel.transferFrom(file.getResource().readableChannel(), start, file.getSize());

            // 关闭文件通道和 RandomAccessFile 对象
            channel.close();
            randomAccessFile.close();

            // 返回上传成功的响应
            return AjaxResult.success("上传成功");
        } catch (Exception e) {
            // 捕获异常并返回上传失败的响应
            return AjaxResult.error("上传失败");
        }
    }

    /**
     * 检测文件是否存在
     * 如果文件存在，则返回已经存在的文件大小。
     * 如果文件不存在，则返回 0，表示前端从头开始上传该文件。
     * @param filename
     * @return
     */
    @GetMapping("/checkFile")
    @ResponseBody
    public AjaxResult checkFile(@RequestParam("filename") String filename) {
        File file = new File(uploadPath+File.separator + filename);
        if (file.exists()) {
            return AjaxResult.success(file.length());
        } else {
            return AjaxResult.success(0L);
        }
    }




    /**
     * 分片上传
     * @param file 文件
     * @param chunkIndex 分片下标
     */
    @PostMapping("/uploadChunk")
    @ResponseBody
    public AjaxResult uploadChunk(@RequestParam("file") MultipartFile file, @RequestParam("chunkIndex") int chunkIndex,@RequestParam("fileName") String fileName) {

        String uploadDirectory = chunkUploadPath+File.separator+fileName;
        File directory = new File(uploadDirectory);
        if (!directory.exists()||directory.isFile()) {
            directory.mkdirs();
        }
        String filePath = uploadDirectory + File.separator + fileName+ "_" + chunkIndex;
        try (OutputStream os = new FileOutputStream(filePath)) {
            os.write(file.getBytes());
            return AjaxResult.success("分片"+(chunkIndex+1)+"上传成功");
        }catch (Exception e){
            // 保存失败后如果文件建立了就删除，下次上传时重新保存，避免文件内容错误
            File chunkFile = new File(filePath);
            if(chunkFile.exists()) chunkFile.delete();
            e.printStackTrace();
            return AjaxResult.error("分片"+(chunkIndex+1)+"上传失败");
        }

    }


    /**
     * 检测分片是否存在
     * 如果文件存在，则返回已经存在的分片下标集合。存在的就不上传
     * 如果文件不存在，则返回空集合，表示前端从头开始上传该文件
     * @param fileName
     * @return
     */
    @GetMapping("/checkChunk")
    @ResponseBody
    public AjaxResult checkChunk(@RequestParam("fileName") String fileName) {
        String uploadDirectory = chunkUploadPath+File.separator+fileName;
        List<Integer> list = new ArrayList<>();
        File file = new File(uploadDirectory);
        // 文件目录不存在
        if(!file.exists()||file.isFile()) return AjaxResult.success(list);

        File[] files = file.listFiles();
        // 文件目录下没有分片文件
        if(files == null) return AjaxResult.success(list);

        // 返回存在分片下标集合
        return AjaxResult.success(Arrays.stream(files).map(item->Integer.valueOf(item.getName().substring(item.getName().lastIndexOf("_")+1))).collect(Collectors.toList()));
    }

    // 合并文件分片
    @PostMapping("/mergeChunks")
    @ResponseBody
    public AjaxResult mergeChunks(@RequestParam("fileName") String fileName, @RequestParam("totalChunks") int totalChunks) {
        String uploadDirectory = chunkUploadPath+File.separator+fileName;
        String mergedFilePath = uploadPath +File.separator+ fileName;

        try (OutputStream os = new FileOutputStream(mergedFilePath, true)) {
            for (int i = 0; i < totalChunks; i++) {
                Path chunkFilePath = Paths.get(uploadDirectory +File.separator+ fileName + "_" + i);
                Files.copy(chunkFilePath, os);
                Files.delete(chunkFilePath);
            }
            File chunkDir = new File(uploadDirectory);
            if (chunkDir.exists()) chunkDir.delete();
            return AjaxResult.success();
        }catch (Exception e){
            e.printStackTrace();
            return AjaxResult.error(e.getMessage());
        }

    }

}

cpp 复制代码

<!DOCTYPE html>
<html lang="zh" xmlns:th="http://www.thymeleaf.org" xmlns:shiro="http://www.pollix.at/thymeleaf/shiro">
<head>
    <th:block th:include="include :: header('分片上传')" />
</head>
<body class="gray-bg">
<div class="container-div" id="chunk-div">
    <div class="row">
        <div class="col-sm-12 search-collapse">
            <form id="formId">
                <div class="select-list">
                    <ul>
                        <li>
                            <label>选择文件：</label>
                            <input type="file" id="fileInput"/>
                        </li>

                        <li>
                            <a class="btn btn-primary btn-rounded btn-sm" @click="startUpload(1)"><i class="fa fa-upload"></i>&nbsp;开始上传1</a>
                            <a class="btn btn-primary btn-rounded btn-sm" @click="startUpload(2)"><i class="fa fa-upload"></i>&nbsp;开始上传2</a>
                        </li>
                    </ul>
                </div>
            </form>
        </div>
        <div class="col-sm-12" style="padding-left: 0;">
            <div class="ibox">
                <div class="ibox-content">
                    <h3>上传进度</h3>
                    <ul class="sortable-list connectList agile-list" v-if="uploadMsg">
                        <li v-for="item in uploadMsg" :class="item.status+'-element'">
                            {{item.title}}
                            <div class="agile-detail">
                                {{item.result}}
                            </div>
                        </li>
                    </ul>
                </div>
            </div>
        </div>
    </div>
</div>
<th:block th:include="include :: footer" />
<script th:inline="javascript">
    var prefix = ctx + "/kuroshiro/file-upload";

    new Vue({
        el: '#chunk-div',
        data: {
            // 每次上传大小
            chunkSize: 100 * 1024 * 1024,
            uploadMsg:{},
            startTime:0,
        },

        methods: {
            /**
             * 开始上传
             */
            startUpload: function(type){
                const fileInput = document.getElementById('fileInput');
                const file = fileInput.files[0];
                if (!file) {
                    alert("请选择文件");
                    return;
                }

                const filename = file.name;
                this.uploadMsg = {};
                this.startTime = (new Date()).getTime();
                Vue.set(this.uploadMsg, 'checkMsg', {
                    title:`文件检测`,
                    result: "检测中... ...",
                    status:"info"
                });
                if(type == 1){
                    this.checkFile(filename).then(start => {
                        this.uploadMsg['checkMsg'].result = `检测成功:已存在文件，大小为 ${start}`
                        this.uploadFile(file, start,Math.min(start + this.chunkSize, file.size));
                    },err => {
                        this.uploadMsg['checkMsg'].result = `检测失败:${err}`
                    })
                }
                if(type == 2){
                    this.checkChunk(filename).then(arr => {
                        this.uploadMsg['checkMsg'].result = `检测成功:已存在文件分片 ${arr.length}`
                        this.uploadChunk(file, arr);
                    },err => {
                        this.uploadMsg['checkMsg'].result = `检测失败:${err}`
                        this.uploadMsg['checkMsg'].status = `info`
                    })
                }
            },
            /**
             * 检查是否上传过
             * @param filename
             * @returns {Promise<unknown>}
             */
            checkFile: function(filename) {
                return this.$fetch(prefix+`/checkFile?filename=${filename}`);
            },
            /**
             * 开始分片上传
             * @param file 文件
             * @param start 开始位置
             * @param end 结束位置
             */
            uploadFile: function(file, start,end) {
                if(start < end){
                    const chunk = file.slice(start, end);
                    const formData = new FormData();
                    formData.append('file', chunk);
                    formData.append('start', start);
                    formData.append('fileName', file.name);

                    Vue.set(this.uploadMsg, 'uploadMsg_'+start, {
                        title:`分片 ${start} - ${end} 上传`,
                        result: "上传中... ...",
                        status:"info"
                    });

                    this.$fetch(prefix+'/fragmentUpload', {
                        method: 'POST',
                        body: formData
                    }).then(response => {
                        this.uploadMsg['uploadMsg_'+start].result = `上传成功`;
                        // 递归调用
                        this.uploadFile(file,end,Math.min(end + this.chunkSize, file.size))
                    },err=>{
                        this.uploadMsg['uploadMsg_'+start].result = `上传失败：${err}`;
                        this.uploadMsg['uploadMsg_'+start].status = `danger`;
                    })
                }else{
                    this.uploadMsg['uploadSuccess'] = {
                        title:`文件已上传`,
                        result:`耗时：`+((new Date()).getTime()-this.startTime),
                        status:"info"
                    };
                }

            },
            /**
             * 切割文件为多个分片
             * @param file
             * @returns {*[]}
             */
            sliceFile: function(file) {
                const chunks = [];
                let offset = 0;

                while (offset < file.size) {
                    const chunk = file.slice(offset, offset + this.chunkSize);
                    chunks.push(chunk);
                    offset += this.chunkSize;
                }

                return chunks;
            },
            /**
             * 检查是否上传过
             * @param filename
             * @returns {Promise<unknown>}
             */
            checkChunk: function(filename) {
                return this.$fetch(prefix+`/checkChunk?fileName=${filename}`);
            },

            /**
             * 开始分片上传
             * @param file 文件
             * @param exists 存在的分片下标
             */
            uploadChunk: function(file,exists) {
                const chunkArr = this.sliceFile(file);
                Promise.all(chunkArr.map(async (chunk, index) => {
                    if (!exists.includes(index)) {
                        const formData = new FormData();
                        formData.append('file', chunk);
                        formData.append('fileName', file.name);
                        formData.append('chunkIndex', index);

                        Vue.set(this.uploadMsg, "upload_" + index, {
                            title: `分片 ${index + 1} 上传`,
                            result: "上传中... ...",
                            status: "info"
                        });
                        return new Promise((resolve, reject) => {
                            this.$fetch(prefix+'/uploadChunk', {
                                method: 'POST',
                                body: formData
                            }).then(res => {
                                resolve(res)
                                this.uploadMsg["upload_"+index].result = "上传成功";
                            },err => {
                                reject(err)
                                this.uploadMsg["upload_"+index].result = err;
                                this.uploadMsg["upload_"+index].status = "danger";
                            });
                        })

                    }
                })).then(uploadRes=> {
                    this.uploadMsg["uploadSuccess"] = {
                        title:`上传成功`,
                        result: "耗时："+((new Date()).getTime()-this.startTime),
                        status:"info"
                    };
                    // 合并分片
                    const formData = new FormData();
                    formData.append('fileName', file.name);
                    formData.append('totalChunks', chunkArr.length);

                    Vue.set(this.uploadMsg, 'mergeChunks', {
                        title:`合并分片`,
                        result: "合并中... ...",
                        status:"info"
                    });
                    this.$fetch(prefix + '/mergeChunks', {
                        method: 'POST',
                        body:formData,
                    }).then(mergeRes=>{
                        this.uploadMsg["mergeChunks"].result = "合并成功";
                    },err => {
                        this.uploadMsg["mergeChunks"].result = `合并失败：${err}`;
                        this.uploadMsg["mergeChunks"].status = "danger";
                    });
                });

            },

            $fetch: function(url,requestInit){
                return new Promise((resolve, reject) => {
                    fetch(url,requestInit).then(response => {
                        if (!response.ok) {
                            throw new Error('请求失败');
                        }
                        return response.json();
                    }).then(data => {
                        if (data.code === 0) {
                            resolve(data.data);
                        } else {
                            reject(data.msg)
                        }
                    }).catch(error => {
                        reject(error)
                    });
                });
            },
        }
    });

</script>
</body>
</html>

文件上传 分片上传

初步实现

后端代码

前端

优化

后端代码

前端代码

文件上传分片上传