大文件切片上传、断点续传（前后端vue3+node.js）

忘ci2024-04-19 17:04

前言

在文件上传过程中，面对大容量的文件往往会遇到上传时间过长、网络不稳定导致的上传失败、服务器无法处理大文件等问题。这时候就可以使用大文件切片上传和断点续传技术解决这些问题。

流程图

前端

Html

html 复制代码 
    <div class="container">
        <input ref="fileInput" type="file" multiple style="display: none"  placeholder="选择文件" @change='fileChange'/>
        <div class="uploadBox" @click="uploadClick">
            +
        </div>
        <progress class="progress" :value="progressValue/totalChunks * 100" max="100"></progress>
    </div>
    <button @click="pauseUpload">暂停上传</button>
    <button @click="resumeUpload">恢复上传</button>
  1. 文件选择
js 复制代码 
const uploadClick = () => {
    fileInput.value.click();
}
const fileChange = async(e) =>{
    const file = e.target.files[0]
    if (!file) {
        return;
    }
    const chunkSize = 1024 * 1024 * 10//切片大小
    totalChunks.value = Math.ceil(file.size / chunkSize)//计算切片数量
    uploadStatus.value = Array(totalChunks.value).fill(0);//初始化上传状态
    worker.postMessage({ file:file, totalChunks:totalChunks.value , chunkSize });向子线程发送消息
}
  1. 由于 File 继承了 Blob 的特性，因此可以使用 Blob.prototype.slice() 方法将大文件切割成小块。再使用spark-md5对分片进行计算得出hash值用做校验。为了避免阻塞用户界面和主线程，这里采用Web Worker多线程计算来处理切片文件的哈希值计算量较大的情况。考虑切片计算量比较大采用增量计算策略 结合 Promise.all() 进行异步优化，以提高计算效率。
hashWork.js 复制代码 
// 引入spark-md5库
self.importScripts('spark-md5.min.js');

// 监听主线程发送的消息
onmessage = async function(e) {
  // 解构获取主线程传递的数据：文件对象、切片数量、切片大小
  const { file, totalChunks, chunkSize} = e.data;
  // 初始化存储数据的对象
  let data = {
    chunks: [], // 存储切片数据
    md5: [], // 存储计算得到的MD5值
    fileName: file.name // 文件名
  };
  // 存储所有计算MD5值的Promise
  let hashPromises = [];
  // 遍历所有切片进行MD5值计算
  for (let i = 0; i < totalChunks; i++) {
    const start = i * chunkSize;
    const end = Math.min((i + 1) * chunkSize, file.size);
    const chunk = file.slice(start, end);
    // 对首尾两个切片以及中部数据块进行不同的处理
    if(i == 0 || i + 1 == totalChunks){
      hashPromises.push(hash(chunk)); // 计算单独切片的MD5值
    }else{
      const head = chunk.slice(0, 5);
      const middle = chunk.slice(Math.floor(end/ 2), Math.floor(chunk.byteLength / 2) + 5)
      const tail = chunk.slice(-5)
      const combinedData = new Blob([head,middle,tail])
      hashPromises.push(hash(combinedData)); // 将头部、中间、尾部数据合并后计算MD5值
    }
    // 将切片数据存入data对象中
    data.chunks.push(chunk);
  }
  // 等待所有MD5值计算完成
  const md5Values = await Promise.all(hashPromises);
  data.md5 = md5Values;
  // 将包含切片数据和MD5值的data对象发送回主线程
  postMessage(data);
};
// 计算给定数据块的MD5值
function hash(chunk) {
  return new Promise((resolve, reject) => {
    const spark = new self.SparkMD5.ArrayBuffer(); // 创建SparkMD5实例
    const fileReader = new FileReader();
    fileReader.onload = (e) => {
      // 读取数据块并计算MD5值
      spark.append(e.target.result);
      resolve(spark.end()); // 返回计算得到的MD5值
    };
    fileReader.readAsArrayBuffer(chunk); // 以ArrayBuffer格式读取数据块
    fileReader.onerror = () => {
      reject(new Error('hash计算错误')); // 发生错误时reject
    };
  });
}
  1. 主线程接收文件切片数组、hash数组、文件名遍历切片数组上传
js 复制代码 
const worker = new Worker(new URL('@/utils/hashWorker.js', import.meta.url))
worker.onmessage = function(e) {
    const { chunks, md5, fileName } = e.data
    chunkData.value = { chunks, md5, fileName }
    chunks.map(( item , index )=>{
        if(shouldStopUpload.value) return;//shouldStopUpload中断循环
        const formData = new FormData();
        formData.append('totalChunks', totalChunks.value)
        formData.append('chunkIndex', index)
        formData.append('md5', md5[index])
        formData.append('chunk', item)
        const signal = controller.value.signal
        request(formData,signal,index)
    })
}
  1. 处理请求、修改上传状态
js 复制代码 
import { fileUpload , margeFile} from '@/api/index'
const request = (formData,signal,index) => {
    fileUpload(formData,signal).then(res =>{
        if(res.status == 200){
            uploadStatus.value[index] = 1 //切片上传成功，修改上传状态数组
            progressValue.value ++
            if(uploadStatus.value.every(num => num === 1)){
            //全部切片上传完毕，请求合并
                const data = {
                    chunksList: chunkData.value.md5,
                    fileName: chunkData.value.fileName
                }
                margeFile(data).then(res =>{
                    if(res.status == 200){
                        alert(res.fileUrl);
                    }
                })
            }
        }
    })
}
  1. 暂停上传，使用AbortController.abort终止请求
ini 复制代码 
const controller = ref(new AbortController())
const shouldStopUpload = ref(false)
const pauseUpload = () => {
    shouldStopUpload.value = true
    controller.value.abort();
}
  1. 续传，因为signal是只读属性，所以AbortController.abort终止请求后无法再次请求，需要new AbortController创建新的实例并更新controller的值来获取新的可用的signal信号
js 复制代码 
const resumeUpload = () => {
    shouldStopUpload.value = false //
    controller.value = new AbortController()
    const signal = controller.value.signal
    uploadStatus.value.map((item,index)=>{//遍历状态数组，上传未上传的切片及上传失败的数组切片
        if(shouldStopUpload.value) return;
        if(item == 0){
            const chunk = chunkData.value.chunks[index]
            const md5 = chunkData.value.md5[index]
            const formData = new FormData();
            formData.append('totalChunks', totalChunks.value)
            formData.append('chunkIndex', index)
            formData.append('md5', md5)
            formData.append('chunk', chunk)
            request(formData,signal,index)
        }
    })
}

后端

  1. 使用multer接收切片文件，multer中间件可以处理multipart/form-data类型的文件上传。注意切片文件要放最后，否则multer无法正确接收放在切片文件后的参数数据
js 复制代码 
const multer = require('multer');
const storage = multer.diskStorage({
  // 设置文件存储目录为 'temporary/'
  destination: function (req, file, cb) {
    cb(null, 'temporary/');
  },
  // 设置文件名为请求体中的md5值
  filename: function (req, file, cb) {
    cb(null, req.body.md5);
  },
  //fileFilter函数筛选如果文件已存在跳过上传
  fileFilter: function(req, file, cb) { 
    const filePath = path.join(__dirname, 'temporary', req.body.md5)
    fs.access(filePath, fs.constants.F_OK, (err) => {
      if (err) {
        cb(null, false)
      }
    });
  }
});

const upload = multer({ storage: storage });
app.post('/upload', upload.single('chunk'), (req, res) => {
    if (req.file) {
      res.status(200).json({ message: '切片上传成功' });
    } else {
      res.status(400).json({ message: '切片上传失败' });
    }
});
  1. 合并切片，成功返回文件地址。writeStream要注意串行写入确保文件合并后可以正常使用，合并完成后关闭写入流。
js 复制代码 
const fs = require('fs');

// 设置'/file'路由为静态文件服务，指向 'files' 目录
app.use('/file', express.static(path.join(__dirname, 'files')));

// 处理文件合并的 POST 请求
app.post('/mergeFile', async (req, res) => {
  const chunks = req.body.chunksList; // 获取切片列表
  const fileName = req.body.fileName; // 获取文件名

  // 检查切片列表是否为空
  if (!chunks || chunks.length === 0) {
    return res.status(400).send('缺少切片列表');
  }

  // 设置合并后保存的文件路径
  const uploadsPath = path.join(__dirname, 'files', fileName);
  const writeStream = fs.createWriteStream(uploadsPath);

  // 异步函数用于合并切片
  async function mergeChunks() {
    for (const file of chunks) {
      const temporaryPath = path.join(__dirname, 'temporary', file);
      await new Promise((resolve, reject) => {
        const readStream = fs.createReadStream(temporaryPath);
        readStream.pipe(writeStream, { end: false });
        readStream.on('end', () => {
          fs.unlinkSync(temporaryPath); // 删除临时切片文件
          resolve();
        });
        readStream.on('error', (err) => {
          reject(err);
        });
      });
    }
  }

  try {
    await mergeChunks(); // 执行切片合并操作
    writeStream.end(); // 关闭写入流
    const fileUrl = `http://127.0.0.1:8080/file/${fileName}`; // 合并后文件的访问路径
    res.status(200).json({ message: '文件合成成功', fileUrl }); // 返回成功信息和文件URL
  } catch (error) {
    writeStream.end(); // 关闭写入流
    res.status(500).json({ message: '文件合成失败', error: error.message }); // 返回错误信息
  }
});
完整代码

前端

upload.vue 复制代码 
<template>
    <div class="container">
        <input ref="fileInput" type="file" multiple style="display: none"  placeholder="选择文件" @change='fileChange'/>
        <div class="uploadBox" @click="uploadClick">
            +
        </div>
        <progress class="progress" :value="progressValue/totalChunks * 100" max="100"></progress>
    </div>
    <button @click="pauseUpload">暂停上传</button>
    <button @click="resumeUpload">恢复上传</button>
</template>


<script setup>
import { ref } from 'vue'
import { fileUpload , margeFile} from '@/api/index'

const worker = new Worker(new URL('@/utils/hashWorker.js', import.meta.url))
const controller = ref(new AbortController())
const uploadStatus = ref()
const shouldStopUpload = ref(false)
const totalChunks = ref(100)
const progressValue = ref(0)
const chunkData= ref()
const fileInput = ref(null);

const uploadClick = () => {
    fileInput.value.click();
}
const fileChange = async(e) =>{
    const file = e.target.files[0]
    if (!file) {
        return;
    }
    const chunkSize = 1024 * 1024 * 10
    totalChunks.value = Math.ceil(file.size / chunkSize)
    uploadStatus.value = Array(totalChunks.value).fill(0);
    worker.postMessage({ file:file, totalChunks:totalChunks.value, chunkSize});
}


const resumeUpload = () => {
    shouldStopUpload.value = false
    controller.value = new AbortController()
    const signal = controller.value.signal
    uploadStatus.value.map((item,index)=>{
        if(shouldStopUpload.value) return;
        if(item == 0){
            const chunk = chunkData.value.chunks[index]
            const md5 = chunkData.value.md5[index]
            const formData = new FormData();
            formData.append('totalChunks', totalChunks.value)
            formData.append('chunkIndex', index)
            formData.append('md5', md5)
            formData.append('chunk', chunk)
            request(formData,signal,index)
        }
    })
}

const pauseUpload = () => {
    shouldStopUpload.value = true
    controller.value.abort();
}

const request = (formData,signal,index) => {
    fileUpload(formData,signal).then(res =>{
        if(res.status == 200){
            uploadStatus.value[index] = 1
            progressValue.value ++
            if(uploadStatus.value.every(num => num === 1)){
                const data = {
                    chunksList: chunkData.value.md5,
                    fileName: chunkData.value.fileName
                }
                margeFile(data).then(res =>{
                    if(res.status == 200){
                        alert(res.fileUrl);
                    }
                })
            }
        }
    })
}

worker.onmessage = function(e) {
    const { chunks, md5, fileName } = e.data
    chunkData.value = { chunks, md5, fileName }
    chunks.map(( item , index )=>{
        if(shouldStopUpload.value) return;
        const formData = new FormData();
        formData.append('totalChunks', totalChunks.value)
        formData.append('chunkIndex', index)
        formData.append('md5', md5[index])
        formData.append('chunk', item)
        const signal = controller.value.signal
        request(formData,signal,index)
    })
}

</script>

<style lang="scss">
.container{
    display: grid;
    place-items: center;
    .uploadBox {
        width: 200px;
        height: 200px;
        display: flex;
        justify-content: center;
        align-items: center;
        border: 1px dashed #c0ccda;
        border-radius: 5px;
    }
    .uploadBox:hover{
        border: 1px dashed #409eff;
    }
    .progress{
        width: 200px;
    }
}
</style>
hashWork.js 复制代码 
self.importScripts('spark-md5.min.js');

onmessage = async function(e) {
  const { file, totalChunks, chunkSize} = e.data;
  let data = {
    chunks: [],
    md5: [],
    fileName: file.name
  };
  let hashPromises = [];
  for (let i = 0; i < totalChunks; i++) {
    const start = i * chunkSize;
    const end = Math.min((i + 1) * chunkSize, file.size);
    const chunk = file.slice(start, end);
    if(i == 0 || i + 1 == totalChunks){
      hashPromises.push(hash(chunk));
    }else{
      const head = chunk.slice(0, 5);
      const middle = chunk.slice(Math.floor(end/ 2), Math.floor(chunk.byteLength / 2) + 5)
      const tail = chunk.slice(-5)
      const combinedData = new Blob([head,middle,tail])
      hashPromises.push(hash(combinedData));
    }
    data.chunks.push(chunk);
  }
  const md5Values = await Promise.all(hashPromises);
  data.md5 = md5Values
  postMessage(data);
};

function hash(chunk) {
  return new Promise((resolve, reject) => {
    const spark = new self.SparkMD5.ArrayBuffer();
    const fileReader = new FileReader();
    fileReader.onload = (e) => {
      spark.append(e.target.result);
      resolve(spark.end());
    };
    fileReader.readAsArrayBuffer(chunk);
    fileReader.onerror = () => {
      reject(new Error('hash计算错误'));
    };
  });
}

后端

node.js 复制代码 
const express = require('express');
const app = express();
const path = require('path')
const fs = require('fs');
const multer = require('multer');

app.use((req, res, next) => {
  res.setHeader('Access-Control-Allow-Origin', '*'); 
  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
  res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
  if (req.method === 'OPTIONS') {
    res.sendStatus(200)
  } else {
    next()
  }
});

const bodyParser = require('body-parser');
app.use(bodyParser.json())
app.use(bodyParser.urlencoded({ extended: false }))
app.use('/file', express.static(path.join(__dirname, 'files')));

const storage = multer.diskStorage({
  destination: function (req, file, cb) {
    cb(null, 'temporary/');
  },
  filename: function (req, file, cb) {
    cb(null, req.body.md5);
  },
  fileFilter: function(req, file, cb) {
    const filePath = path.join(__dirname, 'temporary', req.body.md5)
    fs.access(filePath, fs.constants.F_OK, (err) => {
      if (err) {
        cb(null, false)
      }
    });
  }
});

const upload = multer({ storage: storage });

app.post('/upload', upload.single('chunk'), (req, res) => {
    if (req.file) {
      res.status(200).json({ message: '切片上传成功' });
    } else {
      res.status(400).json({ message: '切片上传失败' });
    }
});

app.post('/mergeFile', async (req, res) => {
  const chunks = req.body.chunksList;
  const fileName = req.body.fileName;
  if (!chunks || chunks.length === 0) {
    return res.status(400).send('缺少切片列表');
  }
  const uploadsPath = path.join(__dirname, 'files', fileName);
  const writeStream = fs.createWriteStream(uploadsPath);
  async function mergeChunks() {
    for (const file of chunks) {
      const temporaryPath = path.join(__dirname, 'temporary', file);
      await new Promise((resolve, reject) => {
        const readStream = fs.createReadStream(temporaryPath);
        readStream.pipe(writeStream, { end: false });
        readStream.on('end', () => {
          fs.unlinkSync(temporaryPath);
          resolve();
        });
        readStream.on('error', (err) => {
          reject(err);
        });
      });
    }
  }
  try {
    await mergeChunks();
    writeStream.end();
    const fileUrl = `http://127.0.0.1:8080/file/${fileName}`
    res.status(200).json({ message: '文件合成成功', fileUrl});
  } catch (error) {
    writeStream.end();
    res.status(500).json({ message: '文件合成失败', error: error.message });
  }
});

app.listen(8080, () => {
    console.log('Server is running on http://127.0.0.1:8080/');
});
相关推荐
程序员爱技术1 小时前
Vue 2 + JavaScript + vue-count-to 集成案例
前端·javascript·vue.js
并不会2 小时前
常见 CSS 选择器用法
前端·css·学习·html·前端开发·css选择器
衣乌安、2 小时前
【CSS】居中样式
前端·css·css3
兔老大的胡萝卜2 小时前
ppk谈JavaScript，悟透JavaScript，精通CSS高级Web，JavaScript DOM编程艺术，高性能JavaScript pdf
前端·javascript
低代码布道师2 小时前
CSS的三个重点
前端·css
耶啵奶膘4 小时前
uniapp-是否删除
linux·前端·uni-app
NiNg_1_2344 小时前
SpringBoot整合SpringSecurity实现密码加密解密、登录认证退出功能
java·spring boot·后端
王哈哈^_^5 小时前
【数据集】【YOLO】【目标检测】交通事故识别数据集 8939 张，YOLO道路事故目标检测实战训练教程！
前端·人工智能·深度学习·yolo·目标检测·计算机视觉·pyqt
Chrikk5 小时前
Go-性能调优实战案例
开发语言·后端·golang
幼儿园老大*5 小时前
Go的环境搭建以及GoLand安装教程
开发语言·经验分享·后端·golang·go
热门推荐
01如何才能让手机厂商主动拥抱华为，接入鸿蒙系统？02【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总03组基轨迹建模 GBTM的介绍与实现（Stata 或 R）04【TC3xx芯片】TC3xx芯片电源管理系统PMS详解05基于YOLOv10深度学习的CT扫描图像肾结石智能检测系统【python源码+Pyqt5界面+数据集+训练代码】深度学习实战、目标检测06【经验分享】Ubuntu22.04安装微信(linux官方版)07Macbook pro M1 安装Ubuntu教程08RAG 实践- Ollama+RagFlow 部署本地知识库09Windows10安装PCL1.14.0及点云配准10文件或文件夹名称中有空格如何批量去除