引言
本文将从零开始，详细拆解一个大文件上传项目（前端部分）的完整开发流程。我们会从项目搭建入手，先实现一个支持暂停、重试、并发控制的任务队列（TaskQueue），再基于此构建具备切片上传、哈希生成、进度监控等功能的上传器（Uploader），最终完成可交互的演示 Demo（建议搭配 《大文件上传服务实现（后端篇）》 食用^_^，或自行搭建后端服务 ）。
核心目录
bash
src
├─App.css
├─App.tsx
├─index.css
├─main.tsx
├─assets
│ react.svg
├─constant
│ index.ts // 项目中涉及常量的声明位置
├─playground
│ taskQueue.tsx // 任务队列演示UI
│ uploader.tsx // 文件上传演示UI
├─uploader
│ index.ts // 入口文件
│ taskQueue.d.ts // 任务队列ts类型声明文件
│ taskQueue.ts // 任务队列功能实现
│ type.ts // 任务队列、文件上传类型入口
│ uploader.d.ts // 文件上传ts类型声明文件
│ uploader.ts // 文件上传功能实现
└─utils
index.ts // 一些工具函数等搭建过程
准备工作
使用 vite 搭建一个文件上传项目
shell
yarn create vite依次填写/选择以下信息
shell
Project name: upload-file-fe
Select a framework: React
Select a variant: Typescript
Use rolldown-vite (Experimental): No
Install with yarn and start now?: Yes实现任务队列
- 支持暂停、重启，可手动终止（手动终止所有功能不可用），
- 支持在执行期间、暂停期间、执行结束时新增新增任务。
- 支持任务执行失败后自动重试
- 支持设置最大重试次数、最大task失败数量、task最大并发数量
任务队列类型声明
路径：src/uploader/taskQueue.d.ts
typescript
export enum TaskStatus {
// 默认状态
Default = 0,
// 准备运行（任务放入 requestIdleCallback 到开始执行这段时间内的状态）
PrepareRun = 1,
// 运行中
Running = 2,
// 重试
Retry = 3,
// 暂停中
Pause = 4,
// 失败
Failded = 5,
// 成功
Success = 6,
}
export type Task<T = unknown> = {
id: number;
status: TaskStatus;
failCount: number;
error: unknown;
result: T;
idleCallbackId?: number;
executor: (task: Task<T>) => Promise<T>;
};
export type QueueTasksRecord = {
failedTasks: Task[];
successTasks: Task[];
runningTasks: Task[];
pendingTasks: Task[];
tasks: Task[];
};
export type QueueTaskConfig = {
/**
* task执行失败后，可重试的次数
*/
maxRetries: number;
/**
* task执行失败后，可重试的次数
*/
maxConcurrency: number;
/**
* 最大失败任务数量
*
* 达到该数量会停止所有任务，队列处理器置为失败状态
*/
maxFailCount: number;
/**
* 待执行的任务队列
*
* 队列实例创建后，也可通过实例的 appendTasks 追加任务
*/
tasks?: Pick<Task, "id" | "executor">[];
/**
* 失败任务数量达到上限或所有任务执行完成（执行完成不代表成功）时触发
*/
onFinish?: (tasksRecord: QueueTasksRecord) => void;
/**
* 任务队列状态变更时触发
*/
onStatusChange?: (status: TaskQueueStatus) => void;
/**
* TODO:任务队列中某个队列（数量、内部某个任务状态）变更时触发
*/
// onChange?: (tasksRecord: QueueTasksRecord) => void;
};
export type QueueTaskContext = Exclude<QueueTaskConfig, "tasks">;
export enum TaskQueueStatus {
// 默认状态
Default = 0,
// 运行中
Running = 1,
// 失败
Failded = 2,
// 成功
Complete = 3,
// 已暂停
Pause = 4,
// 中止
Cancel = 5,
}路径：src/uploader/type.ts
typescript
export * from "./taskQueue.d";任务队列功能实现
准备任务队列中涉及的常量，路径：src/constant/index.ts
typescript
export const DefaultMaxRetries = 2;
export const DefaultMaxConcurrency = 3;
export const DefaultMaxFailCount = 1;路径：src/uploader/taskQueue.tsx
typescript
import {
DefaultMaxConcurrency,
DefaultMaxFailCount,
DefaultMaxRetries,
} from "../constant";
import {
TaskQueueStatus,
TaskStatus,
type QueueTaskConfig,
type QueueTaskContext,
type Task,
} from "./type";
// 所有队列key
const QueueKeys = [
"runningTasks",
"pendingTasks",
"failedTasks",
"successTasks",
"tasks",
] as const;
// 可执行的任务状态
const EnableRunTaskStatus = [
TaskStatus.Retry,
TaskStatus.Default,
TaskStatus.Pause,
];
export class TaskQueue {
// 全部任务
private tasks: Task[] = [];
private taskMap: Record<string, Task> = {};
// 各种队列
private runningTasks: Task[] = []; // 运行中
private pendingTasks: Task[] = []; // 待执行
private failedTasks: Task[] = []; // 失败
private successTasks: Task[] = []; // 成功
private ctx!: QueueTaskContext;
private status!: TaskQueueStatus;
constructor(config: Partial<QueueTaskConfig> = {}) {
this.install(config);
this.updateStatus(TaskQueueStatus.Default);
}
getStatus = () => this.status;
getTaskQueues = () => {
const queues = {} as Record<(typeof QueueKeys)[number], Task[]>;
for (const k of QueueKeys) {
queues[k] = this[k];
}
return queues;
};
private updateStatus = (status: TaskQueueStatus) => {
// TaskQueue 已中止，停止一切行为
if (this.status === TaskQueueStatus.Cancel) {
console.warn("【TaskQueue】TaskQueue已被中止，修改状态失败");
return;
}
if (status === this.status) return;
this.status = status;
this.ctx.onStatusChange?.(status);
if ([TaskQueueStatus.Failded, TaskQueueStatus.Complete].includes(status)) {
const queues = this.getTaskQueues();
this.ctx.onFinish?.(queues);
}
};
private install = async ({
maxRetries = DefaultMaxRetries,
maxConcurrency = DefaultMaxConcurrency,
maxFailCount = DefaultMaxFailCount,
tasks,
...rest
}: Partial<QueueTaskConfig> = {}) => {
this.ctx = {
maxRetries,
maxConcurrency,
maxFailCount,
...rest,
};
if (tasks?.length) this.appendTasks(tasks);
};
private cancelAllIdleRunTask(tasks: Task[]) {
for (const task of tasks) {
if (task.status === TaskStatus.PrepareRun && task.idleCallbackId) {
cancelIdleCallback(task.idleCallbackId);
}
}
}
/**
* 中止任务队列
*/
async cancel() {
this.updateStatus(TaskQueueStatus.Cancel);
this.cancelAllIdleRunTask(this.runningTasks);
}
/**
* 暂停任务队列
*/
pause() {
this.updateStatus(TaskQueueStatus.Pause);
this.cancelAllIdleRunTask(this.runningTasks);
}
start() {
if (!this.pendingTasks.length && !this.runningTasks.length) {
throw new Error(
"【TaskQueue】任务队列不可为空，请在配置中传入 tasks 或通过 appendTasks 添加任务"
);
}
// 避免多个巡检同时进行
if (this.status === TaskQueueStatus.Running) {
console.warn("【TaskQueue】已处于执行状态");
return this;
}
this.updateStatus(TaskQueueStatus.Running);
this.inspectionQueue();
return this;
}
/**
* 新增task，默认支持去重
*/
appendTasks(
tasks: Pick<Task, "id" | "executor">[],
filterExistedTask = true
) {
tasks.forEach(({ id, executor }) => {
const isExisted = !!this.taskMap[id];
if (filterExistedTask && isExisted) return;
if (!filterExistedTask && isExisted)
throw new Error(`【TaskQueue】添加任务失败：任务（${id}）已存在`);
const task = {
id,
status: TaskStatus.Default,
failCount: 0,
executor,
} as Task;
this.tasks.push(task);
this.taskMap[task.id] = task;
this.pendingTasks.push(task);
});
return this;
}
private prepareRun(task: Task) {
task.status = TaskStatus.PrepareRun;
task.idleCallbackId = window.requestIdleCallback(() => this.run(task));
}
/**
* 执行task
* 执行完毕后更新任务状态，巡检任务队列
*/
private async run(task: Task) {
if (this.status !== TaskQueueStatus.Running) {
console.info("【TaskQueue】非执行状态");
return;
}
task.status = TaskStatus.Running;
let isSuccess: boolean;
let error: unknown;
let result: unknown;
try {
result = await task.executor(task);
isSuccess = true;
} catch (err) {
isSuccess = false;
error = err;
}
if (isSuccess) {
// task执行成功
task.status = TaskStatus.Success;
task.result = result;
} else if (!isSuccess && task.failCount < this.ctx.maxRetries) {
// task执行失败重试
task.status = TaskStatus.Retry;
task.failCount++;
delete task.result;
} else {
// task执行失败
console.warn("【TaskQueue】任务执行失败", task);
task.status = TaskStatus.Failded;
task.error = error;
}
this.inspectionQueue();
}
/**
* 巡检 runningTasks、pendingTasks、failedTasks
*/
private inspectionQueue = () => {
const isPauseOrCancel = [
TaskQueueStatus.Pause,
TaskQueueStatus.Cancel,
].includes(this.status);
// taskQueue 已暂停、终止，清空 prepareRunTasks
if (isPauseOrCancel) {
const prepareRunTasks = this.runningTasks.filter(
(e) => e.status === TaskStatus.PrepareRun
);
for (const task of prepareRunTasks) {
task.status = TaskStatus.Pause;
if (task.idleCallbackId) cancelIdleCallback(task.idleCallbackId);
}
console.log("队列已暂停或已终止，巡检停止");
return;
}
const failed: Task[] = [];
const complete: Task[] = [];
const others: Task[] = [];
for (const task of this.runningTasks) {
if (task.status === TaskStatus.Failded) {
this.failedTasks.push(task);
failed.push(task);
} else if (task.status === TaskStatus.Success) {
this.successTasks.push(task);
complete.push(task);
} else {
others.push(task);
}
}
this.runningTasks = others;
// 失败task数量达到上限，触发结束（失败）
if (this.failedTasks.length >= this.ctx.maxFailCount) {
console.log(
"【TaskQueue】失败task数量达到上限",
this.failedTasks,
this.ctx.maxFailCount
);
this.updateStatus(TaskQueueStatus.Failded);
return;
}
// 无可执行任务，触发结束
if (!this.runningTasks.length && !this.pendingTasks.length) {
console.log("【TaskQueue】无可执行任务");
this.updateStatus(TaskQueueStatus.Complete);
return;
}
// runningTasks 存在空缺位置，从 pendingTask 中取出相应数据的 task 补充进去
const vacancies = this.ctx.maxConcurrency - this.runningTasks.length;
if (this.pendingTasks.length && vacancies > 0) {
this.runningTasks.push(...this.pendingTasks.splice(0, 0 + vacancies));
}
// 可准备执行的任务
const prepareRunTask = this.runningTasks.filter((e) =>
EnableRunTaskStatus.includes(e.status)
);
if (prepareRunTask.length) {
for (const task of prepareRunTask) {
this.prepareRun(task);
}
} else {
console.log("【TaskQueue】无可准备执行的任务", this.runningTasks);
}
};
}路径：src/uploader/index.ts
typescript
export * from './taskQueue'
export * from './type'任务队列演示Demo
安装tailwindcss
bash
yarn add @tailwindcss/vite tailwindcss完成vite配置，路径：vite.config.ts
diff
+ import tailwindcss from "@tailwindcss/vite";
export default defineConfig({
+ plugins: [react(), tailwindcss()],
- plugins: [react()],
});路径：src/index.css
css
@import "tailwindcss";路径：src/playground/taskQueue.tsx
typescript
import { useRef, useState } from "react";
import { TaskQueue, TaskQueueStatus } from "../uploader";
export function TaskQueueDemo() {
const lastTaskId = useRef(0);
const taskQueue = useRef<TaskQueue>(null);
const [status, setStatus] = useState<TaskQueueStatus>();
const getTasks = () => {
const tasks = Array.from({ length: 5 }, (_, i) => {
const taskId = lastTaskId.current + i;
return {
id: taskId,
executor: async () => {
console.log(`task(${taskId}) 开始执行`);
return await new Promise((rs) => {
setTimeout(() => {
console.log(`task(${taskId}) 执行完毕`);
rs(taskId);
}, 2000);
});
},
};
});
lastTaskId.current += tasks.length;
return tasks;
};
const handleTaskQueueRun = () => {
if (taskQueue.current) {
const status = taskQueue.current.getStatus();
if (status === TaskQueueStatus.Running) {
alert("当前存在任务队列且处于运行中，请取消后再重新初始化");
return;
}
}
const instance = new TaskQueue({
tasks: getTasks(),
onFinish(tasksRecord) {
console.log("任务队列执行完毕", tasksRecord);
},
onStatusChange: (status) => {
setStatus(status);
},
}).start();
taskQueue.current = instance;
};
const handleTaskQueuePause = () => {
taskQueue.current!.pause();
};
const handleTaskQueueAppend = () => {
taskQueue.current!.appendTasks(getTasks()).start();
};
const handleTaskQueueCancel = () => {
taskQueue.current?.cancel();
taskQueue.current = null;
};
const handleTaskQueueRestart = () => {
taskQueue.current!.start();
};
return (
<>
<div>
<button className="ml-[20px]" onClick={handleTaskQueueRun}>
初始化并执行
</button>
{!!taskQueue.current && (
<>
<button className="ml-[20px]" onClick={handleTaskQueueAppend}>
增加任务并执行
</button>
{status === TaskQueueStatus.Pause && (
<button className="ml-[20px]" onClick={handleTaskQueueRestart}>
重启
</button>
)}
{status === TaskQueueStatus.Running && (
<button className="ml-[20px]" onClick={handleTaskQueuePause}>
暂停
</button>
)}
<button className="ml-[20px]" onClick={handleTaskQueueCancel}>
终止并销毁
</button>
</>
)}
</div>
</>
);
}路径：src/App.tsx
typescript
import "./App.css";
import { TaskQueueDemo } from "./playground/taskQueue";
function App() {
return (
<>
<TaskQueueDemo />
</>
);
}
export default App;实现Uploader
安装ky、crypto-js
bash
yarn add ky crypto-js
yarn add @types/crypto-js -DUploader类型声明
路径：src/uploader/uploader.d.ts
typescript
import { type Options } from "ky";
// 上传器配置
export interface UploaderConfig {
/**
* 上传的文件
*/
file: File;
/**
* 上传服务接口URL
*/
url: string;
/**
* chunk大小
* 默认：1024*3，单位：字节（B）,1024 - 5*1024（1MB-5MB）之间。
*/
chunkSize?: number;
/**
* 上传失败是否重试、重试次数上限
* 默认：3，传入 false / 0 会禁止重试，传入大于0数字会设为重试次数上限
*/
retry?: number | false;
/**
* 自定义请求的入参
*/
getUploadPayload?: (
chunk: ChunkInfo,
index: number
) => {
body?: Options["body"];
searchParams?: Options["searchParams"];
};
/**
* 自定义请求
* 如果 getUploadPayload 满足不了你的需求，postRequest 可以从根本上解决你的问题
*/
postRequest?: () => Promise<UploadTask>;
/**
* 从服务端获取上传文件状态（文件状态、待上传Chunk）
*
*/
getFileStatus?: () => Promise<UploadTask>;
/**
* 上传进度更新时（chunk上传结束）触发
*/
onProgress?: (progress: UploadProgress) => void;
/**
* 上传任务结束时触发
*/
onStatusChange?: (status: UploaderStatus) => void;
}
// 上传器状态
export enum UploaderStatus {
/** 初始化完成 */
Created = 0,
/** 准备生成文件 chunk */
WillGenerateChunks = 1,
/** 文件 chunk 成功 */
ChunksGenerated = 2,
/** 准备生成文件 hash */
WillGenerateHash = 3,
/** 文件 hash 成功 */
HashGenerated = 4,
/** 文件上传中 */
Uploading = 5,
/** 文件上传成功 */
Success = 6,
/**
* 文件上传成功
* 包含以下场景：服务端返回chunk所属文件上传失败、任务队列中失败任务数量达到上限、任务队列所有任务执行完成但存在失败任务
* */
Failed = 7,
/** 已暂停 */
Pause = 8,
}
/**
* 上传进度
*/
export type UploadProgress = {
/**
* 全部chunk数量
*/
totalChunk: number;
/**
* 待上传chunk数量
*/
pendingChunk: number;
/**
* 上传进度百分比
*/
percentage: number;
/**
* 上传速率，单位：字节/秒
*/
speed: number;
};
/**
* 文件hash
*
* 该hash可能是一个非精准的结果。由于文件较大时直接用文件本身生成hash可能存在性能问题，Uplaoder.generate 通过特殊算法生成表示文件hash。
*/
type FileHash = string;
// 上传器上下文
export interface UploaderContext extends UploaderConfig {
hash: FileHash;
chunkSize: number;
// 所有chunk
chunks: ChunkInfo[];
extension?: string;
}
export type ChunkInfo = {
// chunk 下标
index: number;
// 原始File截取的起始下标
start: number;
// 原始File截取的结束下标
end: number;
// 原始File截取得到的chunk
blob: Blob;
};
// api响应体
export type API_Response<T = Record<string, unknown>> = {
code: number;
data: T;
success: boolean;
msg?: string;
};
export type UploadSearchParams = {
// 文件 hash
hash: string;
// chunk 下标
index: number;
// 文件大小
fileSize: number;
// chunk大小
chunkSize: number;
// 文件拓展名
extension?: string;
};
/**
* 文件状态
*/
export enum FileStatus {
/** 上传中 */
Uploading = 0,
/** 合并中 */
Mergeing = 1,
/** 已上传 */
Success = 2,
/** 上传失败 */
Failed = 3,
}
export type UploadTask = {
/** 任务ID（chunk index） */
id: number;
/** 文件hash */
hash: string;
status: FileStatus;
/** 待上传chunks 仅在文件处于"上传中"返回 */
pendingUploadChunks?: number[];
};路径：src/uploader/type.ts
diff
export * from "./taskQueue.d";
+ export * from "./uploader.d";Uploader功能实现
路径：src/constant/index.ts
typescript
export const DefaultFileSizeThreshold = 1024 * 1024 * 500;
// 2MB
export const DefaultChunkSize = 1024 * 1024 * 1;
export const DefaultUploadRetry = 3;
/** file hash crypto key */
export const FILE_HASH_CRYPTO_KEY = "FILE_HASH";路径：src/utils/index.ts
typescript
import CryptoJS from "crypto-js";
import { UploaderStatus } from "../uploader/type";
export async function blobToHash(blob: Blob, algorithm = "SHA256") {
return new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = (r) => {
const result = r.target?.result as ArrayBuffer;
if (!result) {
reject(new Error("【blobToHash】blob读取失败"));
return;
}
try {
// 将 ArrayBuffer 转换为 WordArray
const wordArray = CryptoJS.lib.WordArray.create(result);
let hash;
// 根据算法计算哈希
switch (algorithm.toUpperCase()) {
case "MD5":
hash = CryptoJS.MD5(wordArray);
break;
case "SHA1":
hash = CryptoJS.SHA1(wordArray);
break;
case "SHA256":
hash = CryptoJS.SHA256(wordArray);
break;
case "SHA512":
hash = CryptoJS.SHA512(wordArray);
break;
default:
reject(new Error(`不支持的哈希算法: ${algorithm}`));
return;
}
// 返回十六进制哈希值
resolve(hash.toString(CryptoJS.enc.Hex));
} catch (error) {
reject(error);
}
};
reader.onerror = () => reject(new Error("【blobToHash】blob读取失败"));
reader.readAsArrayBuffer(blob);
});
}
const UploaderTextMap = {
[UploaderStatus.Created]: "初始化完成",
[UploaderStatus.WillGenerateChunks]: "正在生成文件 Chunks",
[UploaderStatus.ChunksGenerated]: "文件 Chunks 生成成功",
[UploaderStatus.WillGenerateHash]: "正在生成文件 Hash",
[UploaderStatus.HashGenerated]: "文件 Hash 生成成功",
[UploaderStatus.Uploading]: "文件上传中",
[UploaderStatus.Failed]: "文件失败",
[UploaderStatus.Success]: "文件成功",
[UploaderStatus.Pause]: "已暂停",
};
export function getUploaderText(status?: UploaderStatus) {
if (status === undefined) return;
return UploaderTextMap[status];
}路径：src/uploader/uploader.ts
typescript
import ky, { type Options } from "ky";
import {
FileStatus,
UploaderStatus,
type API_Response,
type ChunkInfo,
type UploaderConfig,
type UploaderContext,
type UploadProgress,
type UploadSearchParams,
type UploadTask,
} from "./type";
import { TaskQueue } from "./taskQueue";
import {
DefaultChunkSize,
DefaultFileSizeThreshold,
DefaultUploadRetry,
} from "../constant";
import { blobToHash } from "../utils";
const CustomHashChunkCount = 100;
/**
* 支持功能
*
* 1. 上传速度展示
* 2. 上传进度展示
* 3. 单个chunk上传失败重试
* 4. 切片上传
* 5. 秒传
* 6. 断点续传
* 7. 上传并发控制
* 8. 支持暂停/重启
*/
export class Uploader {
private ctx: UploaderContext;
private taskQueue?: TaskQueue;
private status!: UploaderStatus;
constructor({
chunkSize = DefaultChunkSize,
retry = DefaultUploadRetry,
file,
...rest
}: UploaderConfig) {
this.ctx = Object.assign({ chunkSize, retry, file }, rest, {
hash: "",
chunks: [],
});
const names = file.name.split(".");
const extension = names.length > 1 ? names[names.length - 1] : null;
if (extension) {
this.ctx.extension = extension;
}
this._updateStatus(UploaderStatus.Created);
}
private _updateStatus = (status: UploaderStatus, isRestart?: boolean) => {
if (this.status === UploaderStatus.Pause && !isRestart) return;
if (this.status === status) return;
// 文件上传成功或失败，中止任务队列
if ([UploaderStatus.Success, UploaderStatus.Failed].includes(status)) {
this.taskQueue?.cancel();
}
this.ctx.onStatusChange?.(status);
this.status = status;
};
start = async () => {
const startAt = performance.now();
// 生成文件所有chunks
this._updateStatus(UploaderStatus.WillGenerateChunks);
this.ctx.chunks = this.ctx.chunks = await this.generateChunks();
this._updateStatus(UploaderStatus.ChunksGenerated);
const generatedChunksAt = performance.now();
console.log(
"generate chunks success, execution time:",
`${generatedChunksAt - startAt} ms`
);
// 生成hasn
this._updateStatus(UploaderStatus.WillGenerateHash);
this.ctx.hash = await this.generateHash();
this._updateStatus(UploaderStatus.HashGenerated);
const generatedHashAt = performance.now();
console.log(
"generate hash success, execution time:",
`${generatedHashAt - generatedChunksAt} ms`
);
const pendingChunks = await this.getPendingChunk();
if (pendingChunks?.length) {
this.taskQueue = new TaskQueue({
onFinish: (tasks) => {
const { failedTasks } = tasks;
console.log(
"TaskQueue onFinish is emit,but has failedTasks",
failedTasks
);
// 存在失败的任务
if (failedTasks?.length) {
this._updateStatus(UploaderStatus.Failed);
} else {
// 所有任务都成功了，但服务器仍未返回文件上传成功的结果也视为失败
this._updateStatus(UploaderStatus.Failed);
}
},
});
this.appendUploadTask(this.taskQueue, pendingChunks);
this.taskQueue.start();
}
return this;
};
pause = () => {
this._updateStatus(UploaderStatus.Pause);
this.taskQueue?.pause();
};
restart = () => {
// 当Uploader 暂停之后，刚好有个chunk task执行结束返回文件上传成功、失败时
this._updateStatus(UploaderStatus.Uploading, true);
this.taskQueue?.start();
};
getPendingChunk = async () => {
const { getFileStatus = this._getFileStatus } = this.ctx;
try {
const ret = await getFileStatus();
if (ret.status === FileStatus.Success) {
this._updateStatus(UploaderStatus.Success);
} else if (ret.status === FileStatus.Uploading) {
this._updateStatus(UploaderStatus.Uploading);
return ret.pendingUploadChunks;
} else if (ret.status === FileStatus.Mergeing) {
this._updateStatus(UploaderStatus.Uploading);
} else {
// 其他情况视为失败
this._updateStatus(UploaderStatus.Failed);
}
} catch (error) {
console.error("【Uploader】(getPendingChunk) 发生错误", error);
}
};
// 从服务端获取文件的状态
_getFileStatus = async () => {
return await this.postChunk({
chunk: this.ctx.chunks[0],
index: 0,
onProgress: this.ctx.onProgress,
});
};
// 将文件转为chunks
private generateChunks = async () => {
const { file, chunkSize } = this.ctx;
const fr = new FileReader();
return await new Promise<ChunkInfo[]>((resolve) => {
fr.onload = (r) => {
const fl = r.target?.result;
if (!fl) throw new Error("【Uploader】(generateChunks) 读取文件失败");
let start = 0;
const chunks = [] as ChunkInfo[];
while (start < file.size) {
const preStart = start;
const nextEnd = (start += chunkSize);
const blob = file.slice(preStart, nextEnd);
chunks.push({
index: chunks.length,
start: preStart,
end: nextEnd,
blob,
});
}
resolve(chunks);
};
fr.readAsArrayBuffer(file);
});
};
/**
* 生成文件hash
* 由于文件较大时直接用文件本身生成hash可能存在性能问题，这里通过特殊算法生成表示文件的hash
*
* hash的计算：
* - 当文件小于500MB时，文件所有chunks + 文件修改时间
* - 当文件大于等于500MB时，文件部分chunks（第一个、1%、2%、...、99%、最后一个） + 文件修改时间
*
* 实测1.39GB文件，未使用该算法全量chunk计算hash耗时 41960ms，使用该算法后耗时3066ms。
*/
private generateHash = async (
fileSizeThreshold = DefaultFileSizeThreshold
) => {
const { file } = this.ctx;
const fileSize = file.size;
let chunks: ChunkInfo[] = [];
if (fileSize > fileSizeThreshold) {
const indexOffset = Math.floor(
this.ctx.chunks.length / CustomHashChunkCount
);
const chunkIndexs: number[] = Array.from(
{
length: CustomHashChunkCount,
},
(_, i) =>
i === CustomHashChunkCount - 1
? CustomHashChunkCount - 1
: i * indexOffset
);
chunks = chunkIndexs.map((i) => this.ctx.chunks[i]);
} else {
chunks = this.ctx.chunks;
}
const chunkHashs = await Promise.all(chunks.map((e) => blobToHash(e.blob)));
const chunkHashsStr = JSON.stringify({
chunkHashs,
fileLastModified: file.lastModified,
});
const chunksHash = await blobToHash(
new Blob([chunkHashsStr], { type: "application/json" })
);
return chunksHash;
};
private _getUploadPayload = async (chunk: ChunkInfo, index: number) => {
const { chunkSize, file, hash, extension } = this.ctx;
const body = new FormData();
body.append("chunk", chunk.blob);
const searchParams: UploadSearchParams = {
hash: hash,
index: index,
fileSize: file.size,
chunkSize: chunkSize,
extension,
};
return { body, searchParams };
};
private postChunk = async ({
chunk,
index,
onProgress,
}: {
chunk: ChunkInfo;
index: number;
onProgress?: (progress: UploadProgress) => void;
}) => {
const {
getUploadPayload = this._getUploadPayload,
postRequest = this._postRequest,
} = this.ctx;
try {
const { body, searchParams } = await getUploadPayload(chunk, index);
const startAt = performance.now();
const ret = await postRequest(this.ctx.url, body, searchParams);
const endAt = performance.now();
// 更新网速、上传进度
const totalChunk = this.ctx.chunks.length;
const pendingChunk = ret.pendingUploadChunks?.length ?? 0;
const percentage =
Math.floor(((totalChunk - pendingChunk) / totalChunk) * 100 * 100) /
100;
const speed = chunk.blob.size / (endAt - startAt);
onProgress?.({ totalChunk, pendingChunk, percentage, speed });
return ret;
} catch (error) {
return Promise.reject(error);
}
};
private _postRequest = async (
url: string,
body?: Options["body"],
searchParams?: Options["searchParams"]
) => {
try {
const res = await ky.post<API_Response<UploadTask>>(url, {
body,
searchParams: searchParams,
});
const ret = await res.json();
if (ret.success !== true) {
throw new Error("上传失败");
}
return Promise.resolve(ret.data);
} catch (error: unknown) {
return Promise.reject(error);
}
};
private appendUploadTask = async (taskQueue: TaskQueue, chunks: number[]) => {
const tasks = chunks.map((id) => ({
id,
executor: async () => {
try {
const ret = await this.postChunk({
chunk: this.ctx.chunks[id],
index: id,
onProgress: this.ctx.onProgress,
});
if (ret.status === FileStatus.Success) {
this._updateStatus(UploaderStatus.Success);
} else if (ret.status === FileStatus.Uploading) {
this._updateStatus(UploaderStatus.Uploading);
if (ret.pendingUploadChunks?.length) {
this.appendUploadTask(taskQueue, ret.pendingUploadChunks);
}
} else if (ret.status === FileStatus.Mergeing) {
this._updateStatus(UploaderStatus.Uploading);
} else {
this._updateStatus(UploaderStatus.Failed);
}
return ret;
} catch (error) {
return Promise.reject(error);
}
},
}));
taskQueue.appendTasks(tasks);
};
}路径：src/uploader/index.ts
diff
+ export * from './uploader'
export * from './taskQueue'
export * from './type'Uploader演示Demo
路径：src/playground/uploader.tsx
typescript
import { useMemo, useRef, useState, type ChangeEvent } from "react";
import { getUploaderText } from "../utils";
import { Uploader, UploaderStatus, type UploadProgress } from "../uploader";
const URL = "http://127.0.0.1:3010/api/file/upload";
export function UploaderDemo() {
const ref = useRef<File>(null);
const [uploadProgress, setUploadProgress] = useState<UploadProgress | null>(
null
);
const [uploader, setUploader] = useState<Uploader | null>(null);
const [status, setStatus] = useState<UploaderStatus | null>(null);
const { percentage, speed } = uploadProgress || {};
const handleChange = (e: ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
console.log("handleChange", file);
if (!file) return;
setUploader(null);
setStatus(null);
ref.current = file;
};
const handleStart = () => {
if (!ref.current) {
alert("请先添加文件");
return;
}
console.log("file", ref.current);
const uploader = new Uploader({
url: URL,
file: ref.current,
onProgress(progress) {
setUploadProgress(progress);
},
onStatusChange(status) {
console.log("onStatusChange", status);
setStatus(status);
},
});
setUploader(uploader);
uploader.start();
};
const handlePause = () => {
uploader?.pause();
};
const handleRestart = () => {
uploader?.restart();
};
const uploadSpeed = useMemo(() => {
if (typeof speed !== "number") return;
return Math.floor((speed / 1024) * 100) / 100;
}, [speed]);
return (
<>
<div className="controner flex items-center">
<div className="flex-none">
<input type="file" onChange={handleChange} />
</div>
<div className="flex-auto">
{typeof status !== "number" && (
<button onClick={handleStart}>开始</button>
)}
{status === UploaderStatus.Pause && (
<button onClick={handleRestart}>重新开始</button>
)}
{status === UploaderStatus.Uploading && (
<button onClick={handlePause}>暂停</button>
)}
</div>
</div>
<div className="upload-info mt-[20px]">
<div className="flex">
<div>
状态：
{typeof status === "number" ? getUploaderText(status) : "待上传"}
</div>
{status === UploaderStatus.Uploading && (
<div className="ml-[20px]">速率：{uploadSpeed}KB/ms</div>
)}
{typeof status === "number" &&
[
UploaderStatus.Uploading,
UploaderStatus.Failed,
UploaderStatus.Success,
UploaderStatus.Pause,
].includes(status) && (
<div className="ml-[20px]">上传进度：{percentage}%</div>
)}
</div>
</div>
</>
);
}路径：src/App.tsx
diff
import "./App.css";
- import { TaskQueueDemo } from "./playground/taskQueue";
+ // import { TaskQueueDemo } from "./playground/taskQueue";
+ import { UploaderDemo } from "./playground/uploader";
function App() {
return (
<>
- <TaskQueueDemo />
+ {/* <TaskQueueDemo /> */}
+ <UploaderDemo />
</>
);
}
export default App;总结
该大文件上传项目由 TaskQueue、Uploader 两个模块组成，项目中任务执行、暂停、重启、取消、并发控制等管理功能是通过TaskQueue实现的。Uploader实现file分块、hash生成、chunk task处理等功能。
TaskQueue核心方法inspectionQueue会巡检所有任task队列完成更新task状态、分发任务队列、更新taskQueue示例状态、控制并发等操作。
Uploader通过特殊算法选取文件部分 chunk 生成哈希，有效优化了大文件全量 chunk 哈希计算的性能问题 ------ 以 1.3GB 文件为例，耗时从 41 秒大幅降至 3 秒。不过，但这种取部分 chunk 计算出哈希并非精确匹配，存在不同文件被误判为相同的可能性。为此，前端会额外向后台提供文件大小、分块大小、文件扩展名等字段，这些信息共同构成文件 "指纹"，以此进一步提升文件匹配的准确性。