VUE中使用 WebWorker 和 Stream-Saver.JS 实现文件批处理

需求

• 批量下载一批文件，然后打包至一个ZIP文件中。
• 不影响用户其他操作，提升使用体验。

技术要点

• 使用Stream-Saver.JS避免大文件传输导致浏览器内存爆掉，并且可以处理大文件。
• WebWorkrer使用独立的线程，单独的全局上下文，对于计算密集型或高延迟的任务。主线程可以将这些任务分配给 Worker 线程来处理，以提高性能。

实现

安装相关的依赖

如果是在vue项目中使用，这里还需要安装worker-loader。

npm install worker-loader --save-dev
npm install streamsaver --save

配置vue.config.js

//....
chainWebpack: (config) => {
  config.module
      .rule('worker')
      .test(/.worker.js$/)
      .use('worker-loader')
      .loader('worker-loader')
      .options({
        inline: 'fallback'
      });
  config.module.rule('js').exclude.add(/.worker.js$/);
  config.output.globalObject('this');
},
parallel: false
//....

注意，由于这里限制了loader处理的文件类型，webworker的文件只能是.worker.js结尾，例如：xxx.worker.js。

webworker文件

webwoker中没有document，并且由于stream-saver.js使用了它，所以不能在这里处理下载。

发送事件时使用参数来控制事件的不同类型，方便主线程判断。

self.addEventListener('message', event => {
  const data = event.data;
  const type = data.type;
  const downloadItems = data.urls;
  if (type !== 'startDownload') return; //这里需要限制type类型，否则可能会收集到其它的事件。
  let i = 0;
  const urlsLength = downloadItems.length;
  const chunkMap = new Map(); // 收集所有的文件，完成后发送出去。
  downloadItems.forEach(downloadItem => {
    const { url, fileName, fileUid } = downloadItem;
    const name = fileName + '.' + url.split('.').pop();
    fetch(url)
      .then(response => {
        const contentLength = +response.headers.get('Content-Length');
        let receivedLength = 0;
        const reader = response.body.getReader();
        const chunks = []; // 接收到的二进制块的数组（包括 body）
        return reader.read().then(function processStream({ done, value }) {
          if (done) {
            const chunksAll = new Uint8Array(receivedLength);
            let position = 0;
            for (const chunk of chunks) {
              chunksAll.set(chunk, position); 
              position += chunk.length;
            }
            chunkMap.set(fileUid, { name, chunks: chunksAll, type: 'success' });
            self.postMessage({ url, fileUid, complete: true, type: 'downloadSingleSuccess' });
            return;
          }
          chunks.push(value);

          receivedLength += value.length;
          const progress = Math.floor((receivedLength / contentLength) * 100);
          self.postMessage({ url, fileUid, progress, type: 'downloadProgress' }); //发送传输进度

          return reader.read().then(processStream);
        });
      })
      .catch(error => {
        console.log(error.message);
        self.postMessage({ url, fileUid, error: error.message, type: 'downloadError' });
        chunkMap.set(fileUid, { name, chunks: [], type: 'error' });
      }).finally(() => {
        i++;
        if (i === urlsLength) {
          self.postMessage({ url, fileUid, complete: true, value: chunkMap, type: 'downloadSuccess' });
          chunkMap.clear();
        }
      });
  });
});

streamsaver这个依赖的源码中有一个文件zip-stream.js用来将多文件打包成zip ，需要复制出来稍作修改

zip-stream.js

class Crc32 {
  constructor () {
    this.crc = -1;
  }

  append (data) {
    var crc = this.crc | 0; var table = this.table;
    for (var offset = 0, len = data.length | 0; offset < len; offset++) {
      crc = (crc >>> 8) ^ table[(crc ^ data[offset]) & 0xFF];
    }
    this.crc = crc;
  }

  get () {
    return ~this.crc;
  }
}
Crc32.prototype.table = (() => {
  var i; var j; var t; var table = [];
  for (i = 0; i < 256; i++) {
    t = i;
    for (j = 0; j < 8; j++) {
      t = (t & 1)
        ? (t >>> 1) ^ 0xEDB88320
        : t >>> 1;
    }
    table[i] = t;
  }
  return table;
})();

const getDataHelper = byteLength => {
  var uint8 = new Uint8Array(byteLength);
  return {
    array: uint8,
    view: new DataView(uint8.buffer)
  };
};

const pump = zipObj => zipObj.reader.read().then(chunk => {
  if (chunk.done) return zipObj.writeFooter();
  const outputData = chunk.value;
  zipObj.crc.append(outputData);
  zipObj.uncompressedLength += outputData.length;
  zipObj.compressedLength += outputData.length;
  zipObj.ctrl.enqueue(outputData);
});

/**
 * [createWriter description]
 * @param  {Object} underlyingSource [description]
 * @return {Boolean}                  [description]
 */
function createWriter (underlyingSource) {
  const files = Object.create(null);
  const filenames = [];
  const encoder = new TextEncoder();
  let offset = 0;
  let activeZipIndex = 0;
  let ctrl;
  let activeZipObject, closed;

  function next () {
    activeZipIndex++;
    activeZipObject = files[filenames[activeZipIndex]];
    if (activeZipObject) processNextChunk();
    else if (closed) closeZip();
  }

  var zipWriter = {
    enqueue (fileLike) {
      if (closed) throw new TypeError('Cannot enqueue a chunk into a readable stream that is closed or has been requested to be closed');

      let name = fileLike.name.trim();
      const date = new Date(typeof fileLike.lastModified === 'undefined' ? Date.now() : fileLike.lastModified);

      if (fileLike.directory && !name.endsWith('/')) name += '/';
      if (files[name]) throw new Error('File already exists.');

      const nameBuf = encoder.encode(name);
      filenames.push(name);

      const zipObject = files[name] = {
        level: 0,
        ctrl,
        directory: !!fileLike.directory,
        nameBuf,
        comment: encoder.encode(fileLike.comment || ''),
        compressedLength: 0,
        uncompressedLength: 0,
        writeHeader () {
          var header = getDataHelper(26);
          var data = getDataHelper(30 + nameBuf.length);

          zipObject.offset = offset;
          zipObject.header = header;
          if (zipObject.level !== 0 && !zipObject.directory) {
            header.view.setUint16(4, 0x0800);
          }
          header.view.setUint32(0, 0x14000808);
          header.view.setUint16(6, (((date.getHours() << 6) | date.getMinutes()) << 5) | date.getSeconds() / 2, true);
          header.view.setUint16(8, ((((date.getFullYear() - 1980) << 4) | (date.getMonth() + 1)) << 5) | date.getDate(), true);
          header.view.setUint16(22, nameBuf.length, true);
          data.view.setUint32(0, 0x504b0304);
          data.array.set(header.array, 4);
          data.array.set(nameBuf, 30);
          offset += data.array.length;
          ctrl.enqueue(data.array);
        },
        writeFooter () {
          var footer = getDataHelper(16);
          footer.view.setUint32(0, 0x504b0708);

          if (zipObject.crc) {
            zipObject.header.view.setUint32(10, zipObject.crc.get(), true);
            zipObject.header.view.setUint32(14, zipObject.compressedLength, true);
            zipObject.header.view.setUint32(18, zipObject.uncompressedLength, true);
            footer.view.setUint32(4, zipObject.crc.get(), true);
            footer.view.setUint32(8, zipObject.compressedLength, true);
            footer.view.setUint32(12, zipObject.uncompressedLength, true);
          }

          ctrl.enqueue(footer.array);
          offset += zipObject.compressedLength + 16;
          next();
        },
        fileLike
      };

      if (!activeZipObject) {
        activeZipObject = zipObject;
        processNextChunk();
      }
    },
    close () {
      if (closed) throw new TypeError('Cannot close a readable stream that has already been requested to be closed');
      if (!activeZipObject) closeZip();
      closed = true;
    }
  };

  function closeZip () {
    var length = 0;
    var index = 0;
    var indexFilename, file;
    for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) {
      file = files[filenames[indexFilename]];
      length += 46 + file.nameBuf.length + file.comment.length;
    }
    const data = getDataHelper(length + 22);
    for (indexFilename = 0; indexFilename < filenames.length; indexFilename++) {
      file = files[filenames[indexFilename]];
      data.view.setUint32(index, 0x504b0102);
      data.view.setUint16(index + 4, 0x1400);
      data.array.set(file.header.array, index + 6);
      data.view.setUint16(index + 32, file.comment.length, true);
      if (file.directory) {
        data.view.setUint8(index + 38, 0x10);
      }
      data.view.setUint32(index + 42, file.offset, true);
      data.array.set(file.nameBuf, index + 46);
      data.array.set(file.comment, index + 46 + file.nameBuf.length);
      index += 46 + file.nameBuf.length + file.comment.length;
    }
    data.view.setUint32(index, 0x504b0506);
    data.view.setUint16(index + 8, filenames.length, true);
    data.view.setUint16(index + 10, filenames.length, true);
    data.view.setUint32(index + 12, length, true);
    data.view.setUint32(index + 16, offset, true);
    ctrl.enqueue(data.array);
    ctrl.close();
  }

  function processNextChunk () {
    if (!activeZipObject) return;
    if (activeZipObject.directory) return activeZipObject.writeFooter(activeZipObject.writeHeader());
    if (activeZipObject.reader) return pump(activeZipObject);
    if (activeZipObject.fileLike.stream) {
      if (typeof activeZipObject.fileLike.stream !== 'function') {
        activeZipObject.fileLike.stream = new Blob([activeZipObject.fileLike.stream]);
      }
      // activeZipObject.fileLike.stream();
      activeZipObject.crc = new Crc32();
      // --------------**************修改这里, 后面会说到原因***************---------------------
      activeZipObject.reader = activeZipObject.fileLike.stream.stream().getReader();
       // --------------*****************************--------------------
      activeZipObject.writeHeader();
    } else next();
  }
  return new ReadableStream({
    start: c => {
      ctrl = c;
      underlyingSource.start && Promise.resolve(underlyingSource.start(zipWriter));
    },
    pull () {
      return processNextChunk() || (
        underlyingSource.pull &&
                Promise.resolve(underlyingSource.pull(zipWriter))
      );
    }
  });
}

window.ZIP = createWriter; // 注意如果使用的是typescript需要声明一下。

vue文件

<script lang="ts">
//...
import Worker from '@/worker/downloader.worker'; // 使用 work-loader 的好处之一~~~
import StreamSaver from 'streamsaver'; //使用typescript还需要安装 @types/streamsaver
import 'to/path/zip-stream.js'; //这个zip-stream.js的路径需要修改为你的真实路径

// ------data------
urlsToDownload: {
  url: string,
  fileName: string,
  fileUid: string
}[] = [];
worker: Worker | null = null;
// ----------------


async created() {
  this.worker = new Worker();
  this.worker.addEventListener('message', this.handleWorkerMessage); // 注册
}

async beforeDestory() {
  this.worker?.terminate();
}


handleWorkerMessage(e: MessageEvent) {
  const { url, fileName, fileUid, value, error, progress, complete, type } = e.data as {
    url: string;
    fileName: string;
    fileUid: string;
    value: Map<string, {
      chunks: Uint8Array;
      name: string;
      type: 'success' | 'error'
    }>;
    error: string;
    progress: number;
    complete: boolean;
    type: 'downloadProgress' | 'downloadSuccess' | 'downloadError' | 'downloadSingleSuccess';
  };
  if (!type || !['downloadProgress', 'downloadSuccess', 'downloadError', 'downloadSingleSuccess'].includes(type)) return;
  if (progress !== undefined && fileUid && type === 'downloadProgress') {
    // 你可以在这里更新UI来展示某个文件的下载进度
  }
  if (fileUid && type === 'downloadSingleSuccess') {
    // 你可以在这里更新UI来展示某个文件下载成功
  }

  if (e.data.type === 'downloadSuccess') {
    const fileStream = StreamSaver.createWriteStream('archive.zip');

    const readableZipStream = new window.ZIP({
      pull(ctrl: any) {
        value.forEach((v, k) => {
          try {
            if (v.type === 'error') return;
            ctrl.enqueue({ name: v.name, stream: v.chunks });
          } catch (e) {
            console.log(e);
            // 你可以在这里更新 UI 来显示错误信息
          }
        });
        ctrl.close();
      },
    });
    if (window.WritableStream && readableZipStream.pipeTo) {
      readableZipStream.pipeTo(fileStream).then(() => console.log('打包下载成功'));
    }
  }

  if (error && type === 'downloadError') {
    console.error(`Error downloading ${url}: ${error}`);
    // 你可以在这里更新 UI 来显示错误信息
  }
}

startDownload() {
  if (this.worker) {
    this.worker.postMessage({ type: 'startDownload', urls: this.urlsToDownload });
  }
}

//...
</script>