[可乐的随手记 - 6] 并发控制上传大量的文件 (worker + yield )

前言

个人中后期加入项目单机离线项 B端前端 vue2全家桶 后端 Spring + mybatis-plus

本文仅作为自己的记录, 代码非原项目代码, 本文仅作为个人记录

业务场景以及部分截图

前端上传大量的数据表 (未限制文件个数), 后端做数据解析, 提取, 治理, 公司电脑配置: 6700U + 16G内存, Chrome 浏览器

文件上传

原生的Input元素作为上传的载体并隐藏元素

html 复制代码

<input
    multiple
    type="file"
    :accept="accept"
    ref="uploadFileInputElement"
    @change="handleInputValueChange"
/>

<el-button @click="handleClickUploadFile(false)">
    上传文件
</el-button>

<el-button @click="handleClickUploadFile(true)" >
  上传文件夹
</el-button>

点击触发上传文件或者上传文件夹, 以及回调处理

js 复制代码

handleClickUploadFile(folder = false) {
    const fileInputElement = this.$refs.uploadFileInputElement;
    if (!fileInputElement) return;
    triggerUploadFile(fileInputElement, folder);
    folder ? uploadFolder : uploadFile
}

// 移除上传文件夹的实验性属性
const uploadFile = (fileInputElement) => {
    fileInputElement.removeAttribute('webkitdirectory')
}
// 增加上传文件夹的实验性属性
const uploadFolder = (fileInputElement) => {
    fileInputElement.setAttribute('webkitdirectory', true)
}

handleInputValueChange(event) {
  const { files } = event.target;
  const fileList = [...files];
  if (fileList.length) {
    // 显示准备上传的对话框, 界面上先给的一些反馈
    this.$refs.uploadFileProgressRef?.showReadyStatus();
  }
  useQueueUploadFile(fileList, {
    // 子线程过滤文件的回调
    filterFile: ({ index, filterCount, fileTotal }) => {
      this.readyStatusTips = `正在处理第${index}个文件, 共${fileTotal}个文件, ${filterCount}个文件被过滤, 请稍等...`;
    },
    uploadBegin: (needTime) => {
      // 前端假上传的所需时间
      this.totalTime = needTime;
      // 开启假上传进度条
      this.$nextTick(() => {
        this.$refs.uploadFileProgressRef?.startProgress();
      });
    },
    // 上传完成清空input元素的value
    uploadEnd: () => {
      if (this.$refs.uploadFileInputElement) {
        this.$refs.uploadFileInputElement.value = "";
      }
    },
    uploadSuccess: (val) => {
      this.finishCount = val;
      this.$nextTick(() => {
        // 上传完成, 进度条直接完成推进成100%的进度
        this.$refs.uploadFileProgressRef?.doneProgress();
      });
    },
    uploadFail: () => {
      this.finishCount = 0;
      this.$refs.uploadFileProgressRef?.stopProgress();
    },
  });
}

上传控制函数

js 复制代码

// useQueueUploadFile.js
const useQueueUploadFile = async (files, options) => {
  const { uid, filterFile, uploadBegin, uploadEnd, uploadSuccess, uploadFail } = options

  const limit = 6

  let uploadFileCount = 0

  const acceptFiles = ["xls", "xlsx", "csv", "zip", "rar"]

  const awaitUploadFiles = []

  const childrenThreadFilterFileTask = () => {
    return new Promise((resolve) => {
      const { createWorker, destroyWorker } = useWorker()
      const filterWorker = createWorker(useFilterFileInWorkerRoom)
      const fileSizeLimit = 500

      // 触发子线程的任务, 传入文件列表, 过滤的文件类型列表, 单个文件大小上限, 代码就不贴了
      filterWorker.postMessage({
        active: "filterFile",
        params: {
          files,
          acceptFiles,
          fileSizeLimit,
        }
      })

      // 子线程过滤回调
      filterWorker.onmessage = ({ data }) => {
        const { type, result } = data
        switch (type) {
          // 单个文件过滤失败并提示在界面上
          case "filterFileFail":
            const {
              fileName,
              fileSize,
              filterType,
            } = result
            const message = filterType ? `文件: ${fileName}不符合上传类型, 已被过滤上传` : `文件: ${fileName}体积过大(${fileSize}MB), 超过${fileSizeLimit}MB, 已被过滤上传`
            window.requestIdleCallback(() => Message.warning({ message, offset: 320 }))
            break;
          case "filterFileRuning":
            // 过滤正常显示个数同步给主线程进行展示
            filterFile?.(result)
            break;
          case "filterFileSuccess":
            // 文件过滤完成后将文件推入待上传数组并销毁子线程
            awaitUploadFiles.push(...result)
            destroyWorker()
            resolve()
            break;
          default:
            break;
        }
      }
    })
  }

  const mainThreadFilterFileTask = async () => {
    const offset = 320
    return new Promise((resolve) => {
      let tempFile = files[filterFileIndex]
      while (tempFile) {
        const { fileName } = getFileNameAndType(tempFile)
        if (useVerifyFileType(fileName, acceptFiles, { offset }) && !useVerifyFileSize(file, { limit: 500, offset })) {
          awaitUploadFiles.push(tempFile)
        } else {
          filterFileCount++
        }
        filterFile?.({
          index: filterFileIndex + 1,
          filterCount: filterFileCount,
          fileTotal: files.length,
        })
        filterFileIndex++
        tempFile = files[filterFileIndex] || null
        if (!tempFile) {
          resolve()
        }
      }
    })
  }

  window.Worker ? await childrenThreadFilterFileTask() : await mainThreadFilterFileTask()

  const uploadFileTaskEnd = () => {
    uploadEnd?.()
    uploadSuccessCount ? uploadSuccess?.(uploadSuccessCount) : uploadFail?.()
  }

  const awaitUploadFileCount = awaitUploadFiles.length

  if (awaitUploadFileCount === 0) {
    return uploadFileTaskEnd()
  }

  // 生成迭代器
  const iterator = (function* gen() {
    yield* awaitUploadFiles;
  })()

  // 队列上传任务
  const uploadFileTask = async () => {
    const { done, value: file } = iterator.next()

    if (uploadFileCount >= awaitUploadFileCount) {
      return uploadFileTaskEnd()
    }

    if (done) {
      return
    }

    const { fileName, fileType } = getFileNameAndType(file)

    try {
      const formData = new FormData();
      // 复制重新生成文件, 去除在选择文件夹模式下会存在文件夹的路径前缀
      const uploadFile = await copyFile(file, fileName)
      formData.append("file", uploadFile);
      const isZipFile = /(zip|rar)/gi.test(fileType)
      const uploadFunc = isZipFile ? uploadManyFileApi : uploadSingleFileApi ;
      const result = await uploadFunc(formData, uid);

      if (result.code === 200) {
        uploadSuccessCount++
      }
    } catch {
      window.requestIdleCallback(() => {
        Message.error({
          message: `文件上传出现异常, 文件: ${fileName}上传失败`,
          offset: 320,
        });
      })
    } finally {
      uploadFileCount++
    }
    return uploadFileTask()
  }

  // 计算总文件个数所需要的大概时间给假上传进度条
  const needTime = Math.max((awaitUploadFileCount / limit) * uploadFileNeedTime, uploadFileNeedTime)

  uploadBegin?.(needTime)

  // 控制并发数量列为6
  for (let index = 0; index < limit; index++) {
    uploadFileTask()
  }
}