大文件切片上传、断点续传（前后端vue3+node.js）

忘ci2024-04-19 17:04

前言

在文件上传过程中，面对大容量的文件往往会遇到上传时间过长、网络不稳定导致的上传失败、服务器无法处理大文件等问题。这时候就可以使用大文件切片上传和断点续传技术解决这些问题。

流程图

前端

Html

html 复制代码 
    <div class="container">
        <input ref="fileInput" type="file" multiple style="display: none"  placeholder="选择文件" @change='fileChange'/>
        <div class="uploadBox" @click="uploadClick">
            +
        </div>
        <progress class="progress" :value="progressValue/totalChunks * 100" max="100"></progress>
    </div>
    <button @click="pauseUpload">暂停上传</button>
    <button @click="resumeUpload">恢复上传</button>
  1. 文件选择
js 复制代码 
const uploadClick = () => {
    fileInput.value.click();
}
const fileChange = async(e) =>{
    const file = e.target.files[0]
    if (!file) {
        return;
    }
    const chunkSize = 1024 * 1024 * 10//切片大小
    totalChunks.value = Math.ceil(file.size / chunkSize)//计算切片数量
    uploadStatus.value = Array(totalChunks.value).fill(0);//初始化上传状态
    worker.postMessage({ file:file, totalChunks:totalChunks.value , chunkSize });向子线程发送消息
}
  1. 由于 File 继承了 Blob 的特性，因此可以使用 Blob.prototype.slice() 方法将大文件切割成小块。再使用spark-md5对分片进行计算得出hash值用做校验。为了避免阻塞用户界面和主线程，这里采用Web Worker多线程计算来处理切片文件的哈希值计算量较大的情况。考虑切片计算量比较大采用增量计算策略 结合 Promise.all() 进行异步优化，以提高计算效率。
hashWork.js 复制代码 
// 引入spark-md5库
self.importScripts('spark-md5.min.js');

// 监听主线程发送的消息
onmessage = async function(e) {
  // 解构获取主线程传递的数据：文件对象、切片数量、切片大小
  const { file, totalChunks, chunkSize} = e.data;
  // 初始化存储数据的对象
  let data = {
    chunks: [], // 存储切片数据
    md5: [], // 存储计算得到的MD5值
    fileName: file.name // 文件名
  };
  // 存储所有计算MD5值的Promise
  let hashPromises = [];
  // 遍历所有切片进行MD5值计算
  for (let i = 0; i < totalChunks; i++) {
    const start = i * chunkSize;
    const end = Math.min((i + 1) * chunkSize, file.size);
    const chunk = file.slice(start, end);
    // 对首尾两个切片以及中部数据块进行不同的处理
    if(i == 0 || i + 1 == totalChunks){
      hashPromises.push(hash(chunk)); // 计算单独切片的MD5值
    }else{
      const head = chunk.slice(0, 5);
      const middle = chunk.slice(Math.floor(end/ 2), Math.floor(chunk.byteLength / 2) + 5)
      const tail = chunk.slice(-5)
      const combinedData = new Blob([head,middle,tail])
      hashPromises.push(hash(combinedData)); // 将头部、中间、尾部数据合并后计算MD5值
    }
    // 将切片数据存入data对象中
    data.chunks.push(chunk);
  }
  // 等待所有MD5值计算完成
  const md5Values = await Promise.all(hashPromises);
  data.md5 = md5Values;
  // 将包含切片数据和MD5值的data对象发送回主线程
  postMessage(data);
};
// 计算给定数据块的MD5值
function hash(chunk) {
  return new Promise((resolve, reject) => {
    const spark = new self.SparkMD5.ArrayBuffer(); // 创建SparkMD5实例
    const fileReader = new FileReader();
    fileReader.onload = (e) => {
      // 读取数据块并计算MD5值
      spark.append(e.target.result);
      resolve(spark.end()); // 返回计算得到的MD5值
    };
    fileReader.readAsArrayBuffer(chunk); // 以ArrayBuffer格式读取数据块
    fileReader.onerror = () => {
      reject(new Error('hash计算错误')); // 发生错误时reject
    };
  });
}
  1. 主线程接收文件切片数组、hash数组、文件名遍历切片数组上传
js 复制代码 
const worker = new Worker(new URL('@/utils/hashWorker.js', import.meta.url))
worker.onmessage = function(e) {
    const { chunks, md5, fileName } = e.data
    chunkData.value = { chunks, md5, fileName }
    chunks.map(( item , index )=>{
        if(shouldStopUpload.value) return;//shouldStopUpload中断循环
        const formData = new FormData();
        formData.append('totalChunks', totalChunks.value)
        formData.append('chunkIndex', index)
        formData.append('md5', md5[index])
        formData.append('chunk', item)
        const signal = controller.value.signal
        request(formData,signal,index)
    })
}
  1. 处理请求、修改上传状态
js 复制代码 
import { fileUpload , margeFile} from '@/api/index'
const request = (formData,signal,index) => {
    fileUpload(formData,signal).then(res =>{
        if(res.status == 200){
            uploadStatus.value[index] = 1 //切片上传成功，修改上传状态数组
            progressValue.value ++
            if(uploadStatus.value.every(num => num === 1)){
            //全部切片上传完毕，请求合并
                const data = {
                    chunksList: chunkData.value.md5,
                    fileName: chunkData.value.fileName
                }
                margeFile(data).then(res =>{
                    if(res.status == 200){
                        alert(res.fileUrl);
                    }
                })
            }
        }
    })
}
  1. 暂停上传，使用AbortController.abort终止请求
ini 复制代码 
const controller = ref(new AbortController())
const shouldStopUpload = ref(false)
const pauseUpload = () => {
    shouldStopUpload.value = true
    controller.value.abort();
}
  1. 续传，因为signal是只读属性，所以AbortController.abort终止请求后无法再次请求，需要new AbortController创建新的实例并更新controller的值来获取新的可用的signal信号
js 复制代码 
const resumeUpload = () => {
    shouldStopUpload.value = false //
    controller.value = new AbortController()
    const signal = controller.value.signal
    uploadStatus.value.map((item,index)=>{//遍历状态数组，上传未上传的切片及上传失败的数组切片
        if(shouldStopUpload.value) return;
        if(item == 0){
            const chunk = chunkData.value.chunks[index]
            const md5 = chunkData.value.md5[index]
            const formData = new FormData();
            formData.append('totalChunks', totalChunks.value)
            formData.append('chunkIndex', index)
            formData.append('md5', md5)
            formData.append('chunk', chunk)
            request(formData,signal,index)
        }
    })
}

后端

  1. 使用multer接收切片文件，multer中间件可以处理multipart/form-data类型的文件上传。注意切片文件要放最后，否则multer无法正确接收放在切片文件后的参数数据
js 复制代码 
const multer = require('multer');
const storage = multer.diskStorage({
  // 设置文件存储目录为 'temporary/'
  destination: function (req, file, cb) {
    cb(null, 'temporary/');
  },
  // 设置文件名为请求体中的md5值
  filename: function (req, file, cb) {
    cb(null, req.body.md5);
  },
  //fileFilter函数筛选如果文件已存在跳过上传
  fileFilter: function(req, file, cb) { 
    const filePath = path.join(__dirname, 'temporary', req.body.md5)
    fs.access(filePath, fs.constants.F_OK, (err) => {
      if (err) {
        cb(null, false)
      }
    });
  }
});

const upload = multer({ storage: storage });
app.post('/upload', upload.single('chunk'), (req, res) => {
    if (req.file) {
      res.status(200).json({ message: '切片上传成功' });
    } else {
      res.status(400).json({ message: '切片上传失败' });
    }
});
  1. 合并切片，成功返回文件地址。writeStream要注意串行写入确保文件合并后可以正常使用，合并完成后关闭写入流。
js 复制代码 
const fs = require('fs');

// 设置'/file'路由为静态文件服务，指向 'files' 目录
app.use('/file', express.static(path.join(__dirname, 'files')));

// 处理文件合并的 POST 请求
app.post('/mergeFile', async (req, res) => {
  const chunks = req.body.chunksList; // 获取切片列表
  const fileName = req.body.fileName; // 获取文件名

  // 检查切片列表是否为空
  if (!chunks || chunks.length === 0) {
    return res.status(400).send('缺少切片列表');
  }

  // 设置合并后保存的文件路径
  const uploadsPath = path.join(__dirname, 'files', fileName);
  const writeStream = fs.createWriteStream(uploadsPath);

  // 异步函数用于合并切片
  async function mergeChunks() {
    for (const file of chunks) {
      const temporaryPath = path.join(__dirname, 'temporary', file);
      await new Promise((resolve, reject) => {
        const readStream = fs.createReadStream(temporaryPath);
        readStream.pipe(writeStream, { end: false });
        readStream.on('end', () => {
          fs.unlinkSync(temporaryPath); // 删除临时切片文件
          resolve();
        });
        readStream.on('error', (err) => {
          reject(err);
        });
      });
    }
  }

  try {
    await mergeChunks(); // 执行切片合并操作
    writeStream.end(); // 关闭写入流
    const fileUrl = `http://127.0.0.1:8080/file/${fileName}`; // 合并后文件的访问路径
    res.status(200).json({ message: '文件合成成功', fileUrl }); // 返回成功信息和文件URL
  } catch (error) {
    writeStream.end(); // 关闭写入流
    res.status(500).json({ message: '文件合成失败', error: error.message }); // 返回错误信息
  }
});
完整代码

前端

upload.vue 复制代码 
<template>
    <div class="container">
        <input ref="fileInput" type="file" multiple style="display: none"  placeholder="选择文件" @change='fileChange'/>
        <div class="uploadBox" @click="uploadClick">
            +
        </div>
        <progress class="progress" :value="progressValue/totalChunks * 100" max="100"></progress>
    </div>
    <button @click="pauseUpload">暂停上传</button>
    <button @click="resumeUpload">恢复上传</button>
</template>


<script setup>
import { ref } from 'vue'
import { fileUpload , margeFile} from '@/api/index'

const worker = new Worker(new URL('@/utils/hashWorker.js', import.meta.url))
const controller = ref(new AbortController())
const uploadStatus = ref()
const shouldStopUpload = ref(false)
const totalChunks = ref(100)
const progressValue = ref(0)
const chunkData= ref()
const fileInput = ref(null);

const uploadClick = () => {
    fileInput.value.click();
}
const fileChange = async(e) =>{
    const file = e.target.files[0]
    if (!file) {
        return;
    }
    const chunkSize = 1024 * 1024 * 10
    totalChunks.value = Math.ceil(file.size / chunkSize)
    uploadStatus.value = Array(totalChunks.value).fill(0);
    worker.postMessage({ file:file, totalChunks:totalChunks.value, chunkSize});
}


const resumeUpload = () => {
    shouldStopUpload.value = false
    controller.value = new AbortController()
    const signal = controller.value.signal
    uploadStatus.value.map((item,index)=>{
        if(shouldStopUpload.value) return;
        if(item == 0){
            const chunk = chunkData.value.chunks[index]
            const md5 = chunkData.value.md5[index]
            const formData = new FormData();
            formData.append('totalChunks', totalChunks.value)
            formData.append('chunkIndex', index)
            formData.append('md5', md5)
            formData.append('chunk', chunk)
            request(formData,signal,index)
        }
    })
}

const pauseUpload = () => {
    shouldStopUpload.value = true
    controller.value.abort();
}

const request = (formData,signal,index) => {
    fileUpload(formData,signal).then(res =>{
        if(res.status == 200){
            uploadStatus.value[index] = 1
            progressValue.value ++
            if(uploadStatus.value.every(num => num === 1)){
                const data = {
                    chunksList: chunkData.value.md5,
                    fileName: chunkData.value.fileName
                }
                margeFile(data).then(res =>{
                    if(res.status == 200){
                        alert(res.fileUrl);
                    }
                })
            }
        }
    })
}

worker.onmessage = function(e) {
    const { chunks, md5, fileName } = e.data
    chunkData.value = { chunks, md5, fileName }
    chunks.map(( item , index )=>{
        if(shouldStopUpload.value) return;
        const formData = new FormData();
        formData.append('totalChunks', totalChunks.value)
        formData.append('chunkIndex', index)
        formData.append('md5', md5[index])
        formData.append('chunk', item)
        const signal = controller.value.signal
        request(formData,signal,index)
    })
}

</script>

<style lang="scss">
.container{
    display: grid;
    place-items: center;
    .uploadBox {
        width: 200px;
        height: 200px;
        display: flex;
        justify-content: center;
        align-items: center;
        border: 1px dashed #c0ccda;
        border-radius: 5px;
    }
    .uploadBox:hover{
        border: 1px dashed #409eff;
    }
    .progress{
        width: 200px;
    }
}
</style>
hashWork.js 复制代码 
self.importScripts('spark-md5.min.js');

onmessage = async function(e) {
  const { file, totalChunks, chunkSize} = e.data;
  let data = {
    chunks: [],
    md5: [],
    fileName: file.name
  };
  let hashPromises = [];
  for (let i = 0; i < totalChunks; i++) {
    const start = i * chunkSize;
    const end = Math.min((i + 1) * chunkSize, file.size);
    const chunk = file.slice(start, end);
    if(i == 0 || i + 1 == totalChunks){
      hashPromises.push(hash(chunk));
    }else{
      const head = chunk.slice(0, 5);
      const middle = chunk.slice(Math.floor(end/ 2), Math.floor(chunk.byteLength / 2) + 5)
      const tail = chunk.slice(-5)
      const combinedData = new Blob([head,middle,tail])
      hashPromises.push(hash(combinedData));
    }
    data.chunks.push(chunk);
  }
  const md5Values = await Promise.all(hashPromises);
  data.md5 = md5Values
  postMessage(data);
};

function hash(chunk) {
  return new Promise((resolve, reject) => {
    const spark = new self.SparkMD5.ArrayBuffer();
    const fileReader = new FileReader();
    fileReader.onload = (e) => {
      spark.append(e.target.result);
      resolve(spark.end());
    };
    fileReader.readAsArrayBuffer(chunk);
    fileReader.onerror = () => {
      reject(new Error('hash计算错误'));
    };
  });
}

后端

node.js 复制代码 
const express = require('express');
const app = express();
const path = require('path')
const fs = require('fs');
const multer = require('multer');

app.use((req, res, next) => {
  res.setHeader('Access-Control-Allow-Origin', '*'); 
  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
  if (req.method === 'OPTIONS') {
    res.sendStatus(200)
  } else {
    next()
  }
});

const bodyParser = require('body-parser');
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: false }))
app.use('/file', express.static(path.join(__dirname, 'files')));

const storage = multer.diskStorage({
  destination: function (req, file, cb) {
    cb(null, 'temporary/');
  },
  filename: function (req, file, cb) {
    cb(null, req.body.md5);
  },
  fileFilter: function(req, file, cb) {
    const filePath = path.join(__dirname, 'temporary', req.body.md5)
    fs.access(filePath, fs.constants.F_OK, (err) => {
      if (err) {
        cb(null, false)
      }
    });
  }
});

const upload = multer({ storage: storage });

app.post('/upload', upload.single('chunk'), (req, res) => {
    if (req.file) {
      res.status(200).json({ message: '切片上传成功' });
    } else {
      res.status(400).json({ message: '切片上传失败' });
    }
});

app.post('/mergeFile', async (req, res) => {
  const chunks = req.body.chunksList;
  const fileName = req.body.fileName;
  if (!chunks || chunks.length === 0) {
    return res.status(400).send('缺少切片列表');
  }
  const uploadsPath = path.join(__dirname, 'files', fileName);
  const writeStream = fs.createWriteStream(uploadsPath);
  async function mergeChunks() {
    for (const file of chunks) {
      const temporaryPath = path.join(__dirname, 'temporary', file);
      await new Promise((resolve, reject) => {
        const readStream = fs.createReadStream(temporaryPath);
        readStream.pipe(writeStream, { end: false });
        readStream.on('end', () => {
          fs.unlinkSync(temporaryPath);
          resolve();
        });
        readStream.on('error', (err) => {
          reject(err);
        });
      });
    }
  }
  try {
    await mergeChunks();
    writeStream.end();
    const fileUrl = `http://127.0.0.1:8080/file/${fileName}`
    res.status(200).json({ message: '文件合成成功', fileUrl});
  } catch (error) {
    writeStream.end();
    res.status(500).json({ message: '文件合成失败', error: error.message });
  }
});

app.listen(8080, () => {
    console.log('Server is running on http://127.0.0.1:8080/');
});
相关推荐
卸任6 分钟前
Electron霸屏功能总结
前端·react.js·electron
fengci.6 分钟前
ctfshow黑盒测试前半部分
前端
喵个咪10 分钟前
go-wind-cms 微服务架构设计：为什么基于 Kratos？
后端·微服务·cms
神奇小汤圆16 分钟前
百度面试官：Redis 内存满了怎么办？你有想过吗？
后端
喵个咪18 分钟前
Headless 架构优势：内容与展示解耦，一套 API 打通全端生态
前端·后端·cms
开心就好202519 分钟前
HTTPS超文本传输安全协议全面解析与工作原理
后端·ios
小江的记录本22 分钟前
【JEECG Boot】 JEECG Boot——数据字典管理 系统性知识体系全解析
java·前端·spring boot·后端·spring·spring cloud·mybatis
神奇小汤圆23 分钟前
Spring Batch实战
后端
喵个咪25 分钟前
传统 CMS 太笨重？试试 Headless 架构的 GoWind，轻量又强大
前端·后端·cms
热门推荐
01GitHub 镜像站点02一周AI热点速览（2026.03.31-04.06）：GPT-6曝光、谷歌开源Gemma 4、资本狂飙与模型军备竞赛03OpenClaw 请求超时 llm request timed out 怎么解决？3 种方案实测，附完整排查流程04AI 编程效率翻倍：Superpowers Skills 上手清单 + 完整指南05VMware Workstation Pro 17 虚拟机完整安装教程（2026最新）06Qwen3.5-Omni与Qwen3.6模型全面解析（含测评/案例/使用教程）07Oh My Codex 快速使用指南08UV安装并设置国内源09Claude Code 未登录 使用第三方模型10【Vulhub】Fastjson 1.2.24_rce复现