为什么要做前端导出？
需求一：用户需要导出30W+数据量
前面说过，用户的数据量比较大，又由于是全量数据展示【这里已经做过数据结构优化和压缩】。30W+数据配合虚拟表格前端渲染压力不大，压力在于页面第一次加载的时候比较慢。
那我们直接把数据从内存中转成Blob对象导出也就可以了。
问题来了，前端展示的数据存在加密数据，需要解密导出。纳尼，又要重新去拿解密之后的数据。
行吧，把30W条数据分批次去获取解密数据，每次1000条，一共300个请求。很好，服务器直接打挂了。
加个请求队列吧！！！控制下最大并发数。让它每次最多发6个，当前队列中有请求回来之后立即执行后续队列中请求。
javascript
export default class AsyncRequestQueue {
private maxConcurrency: number;
private queue: Array<{
request: () => Promise<any>;
resolve: (value: unknown) => void;
reject: (value: unknown) => void;
index: number;
processed: boolean;
}>;
private activeCount: number;
private results: Array<any>;
private nextPromiseResolver: any;
constructor(maxConcurrency) {
this.maxConcurrency = maxConcurrency;
this.queue = [];
this.activeCount = 0;
this.results = [];
this.nextPromiseResolver = null;
}
addRequest(request) {
return new Promise((resolve, reject) => {
const requestObj = {
request,
resolve,
reject,
index: this.queue.length,
processed: false,
};
this.queue.push(requestObj);
this.processQueue();
});
}
processQueue() {
if (this.activeCount >= this.maxConcurrency) {
return;
}
const requestObj = this.queue.find(req => !req.processed);
if (!requestObj) {
if (this.activeCount === 0 && this.nextPromiseResolver) {
// 所有请求都已完成
this.nextPromiseResolver(this.results);
}
return;
}
const { request, resolve, reject, index } = requestObj;
requestObj.processed = true;
this.activeCount++;
request()
.then(response => {
this.results.push({ index, response });
resolve(response);
})
.catch(error => {
reject(error);
})
.finally(() => {
this.activeCount--;
this.processQueue();
});
}
execute() {
return new Promise(resolve => {
this.nextPromiseResolver = resolve;
this.processQueue();
});
}
}
javascript
const requestQueue = new AsyncRequestQueue(6);
let selectedKeysChunks = chunk(selectedKeys, MAX_BATCH_TOTAL); // 每1000条数据一个请求批次
promises = selectedKeysChunks.map(selectedKeys => () =>
request<{ data: any }, any>(QUERY_WORK_ORDER_LIST_V3, {}).then(res => res?.data || []))
promises.map(item => requestQueue.addRequest(item));
const list = await requestQueue
.execute()
.then(responses => {
// 按照输入队列的顺序返回请求结果
const orderedResponses = (responses as any[]).sort(
(a, b) => a.index - b.index,
);
return orderedResponses.map(res => res.response);
})
.catch(error => {
console.error('Error occurred during request queue:', error);
});
需求二：列头需要支持多级列头/合并与拆分
表格中的数据都是来自于自定义组件，自定义组件的结构可能是多层级的，所以展示的时候，有的需要合并在一起，有的需要单独拆开进行导出。
- 先通过exceljs建立一个workbook 实例对象
- 列头数据格转换
- 计算列头最大深度，有的列头是3级，有的列头是4级，需要用最深的列头作为标准
- 计算生成Cell单元
- 左右比对，上下比对，如果发现key一致的Cell单元进行合并
typescript
// TODO: fields 结构为[{title: "列头", dataIndex: 'col1', children: []}]
export interface ITableHeader {
header: string;
// 用于数据匹配的 key
key: string;
// 列宽
width: number;
// 父级的 key
parentKey?: string;
children?: ITableHeader[];
}
function generateHeaders(columns: any[], parentIndex?) {
return columns?.map(col => {
const obj: ITableHeader = {
// 显示的 name
header: col.title,
// 用于数据匹配的 key
key: col.dataIndex,
// 列宽
width: DEFAULT_COLUMN_WIDTH,
};
if (parentIndex) {
obj.parentKey = parentIndex;
}
if (Array.isArray(col.children)) {
obj.children = generateHeaders(col.children, col.dataIndex);
}
return obj;
});
}
function calculateMaxDepth(columns: Array<Header>) {
let maxDepth = 1;
function calculateDepth(column, depth) {
if (column.children) {
for (const child of column.children) {
calculateDepth(child, depth + 1);
}
maxDepth = Math.max(maxDepth, depth + 1);
}
}
for (const column of columns) {
calculateDepth(column, 1);
}
return maxDepth;
}
// 计算组件所有叶子节点个数
const getMaxChildLength = node => {
if (!node.children || node.children.length === 0) {
return 1; // 当前节点是叶子节点
}
let count = 0;
for (const childNode of node.children) {
count += getMaxChildLength(childNode);
}
return count;
};
const loopPushCell = (
column,
rowArray: Array<Array<{ title: string; dataIndex: string }>> = [[]],
index = 0,
maxDepth = 0,
) => {
if (index >= maxDepth) return;
const count = getMaxChildLength(column);
const columns = new Array(count)?.fill(null)?.map(() => ({
title: column.title,
dataIndex: column.dataIndex,
}));
rowArray[index].push(...columns);
// 有子节点的时候，继续
if (Reflect.has(column, 'children')) {
++index;
column.children?.map(item => loopPushCell(item, rowArray, index, maxDepth));
} else {
// 无子节点的时候，子节点的数据继承父节点数据
loopPushCell(column, rowArray, ++index, maxDepth);
}
};
// 列头单行合并
export const mergeColumnsCell = (
rowData: Array<{ title: string; dataIndex: string }>,
worksheet: Worksheet,
rowNumber: number,
rowInstance: Row,
) => {
// 当前列指针
let pointer = -1;
rowData.forEach((cellData, index) => {
if (index <= pointer) return;
pointer = lastIndexOfObject(rowData, cellData);
if (index !== pointer) {
const START = getExcelAlpha(index + 1);
const END = getExcelAlpha(pointer + 1);
console.debug(
'mergeColumns',
Number(rowNumber),
index + 1,
Number(rowNumber),
pointer + 1,
`${START}${Number(rowNumber) + 1}:${END}${Number(rowNumber) + 1}`,
);
worksheet.mergeCells(
`${START}${Number(rowNumber) + 1}:${END}${Number(rowNumber) + 1}`,
);
const cell = rowInstance.getCell(index + 1);
cell.alignment = {
vertical: 'middle',
horizontal: 'center',
wrapText: true,
};
}
});
};
// 矩阵反转
export function transposeMatrix<T>(matrix: Array<Array<T>>) {
const numRows = matrix.length;
const numCols = matrix[0].length;
const result: Array<Array<T>> = [];
for (let col = 0; col < numCols; col++) {
const newRow: Array<T> = [];
for (let row = 0; row < numRows; row++) {
newRow.push(matrix[row][col]);
}
result.push(newRow);
}
return result;
}
function drawExcelHeader(
worksheet: Worksheet,
headerRows: Array<Array<{ title: string; dataIndex: string }>>,
) {
// 行处理
headerRows?.map((row, index) => {
// 获取单行列头所有名称
const columnsName = row?.map(columns => columns.title);
// 单行列头插入
const rowHeader = worksheet.addRow(columnsName);
// 样式修改
addHeaderStyle(rowHeader, { color: 'dff8ff' });
// 行合并列头
mergeColumnsCell(row, worksheet, index, rowHeader);
});
if (headerRows?.length > 1) {
// 二维数组反转
const transList = transposeMatrix(headerRows);
transList?.map((row, index) => {
// 列处理
mergeRowCells(row, worksheet, index);
});
}
}
// 创建工作簿
const workbook = new ExcelJs.Workbook();
// 添加sheet
const worksheet = workbook.addWorksheet(fileName);
// 设置 sheet 的默认行高
worksheet.properties.defaultRowHeight = 20;
// 解析 columns
const headers = generateHeaders(fields);
// 计算当前Excel列头最大深度
const maxDepth = calculateMaxDepth(fields);
// 根据最大深度，生成需要的二维数组，每个元素对应一个Cell格
const excelHeaderRow: Array<Array<{
title: string;
dataIndex: string;
}>> = new Array(maxDepth).fill(null).map(() => []);
fields?.map(item => loopPushCell(item, excelHeaderRow, 0, maxDepth));
drawExcelHeader(worksheet, excelHeaderRow);需求三：图片导出
用户说针对图片类型的数据，我不想在Excel中展示链接，我想看图片。。。
- 先从列头中找到哪些列是图片，这里列头里面有组件类型，可以根据组件类型来找到
- 找出每一条数据中最大图片长度，根据最大长度扩展列头长度
- 为列头增加dataIndex
- 图片写入
typescript
// TODO: 列头计算
const formatPictures = pictures => {
if (!pictures) return [];
pictures = pictures.replace(/[\[\]]/g, '');
pictures = pictures.replace(/\s/g, '');
pictures = pictures ? pictures.split(',') : [];
return pictures.map(
item => `https://kefu.kuaimai.com/${item}?x-oss-process=image/resize,l_500`,
);
};
export const exportPicture = ({ columns, dataSource, filterColumns }) => {
const pictures = columns.filter(column => column.type === 'PICTURE');
if (pictures?.length > 0) {
// 获取所有的图片组件的组件id
const picturesKeys = pictures.map(item => `${item.dataIndex}_picture`);
// 为每个图片组件确认图片个数
let uniKeyMaxPic: picturesKeysType = picturesKeys.reduce((cur, nxt) => {
cur[nxt] = 1;
return cur;
}, {});
uniKeyMaxPic = dataSource.reduce((cur, nxt) => {
Object.keys(cur).map(i => {
cur[i] = Math.max(cur[i], formatPictures(nxt[i])?.length);
});
return cur;
}, uniKeyMaxPic);
// 采集最大单行图片个数
const maxPicture = Object.values(uniKeyMaxPic).reduce(
(cur: number, nxt) => {
return (cur += Number(nxt));
},
0,
);
if (maxPicture > 0) {
// 开始对导出列头新增图片列
filterColumns = filterColumns.filter(
item => !picturesKeys.includes(`${item.dataIndex}_picture`),
);
const finalUniKeyMaxPic = Object.keys(uniKeyMaxPic).reduce((cur, nxt) => {
return {
...cur,
[nxt]: {
dataIndex: new Array(uniKeyMaxPic[nxt])
.fill('')
.map(item => uuidv4()),
title: columns.find(c => nxt?.includes(c.dataIndex))?.name,
},
};
}, {});
let flatColumns = [];
Object.keys(finalUniKeyMaxPic).map(nxt => {
if (finalUniKeyMaxPic[nxt].dataIndex?.length > 0) {
flatColumns = flatColumns.concat(
finalUniKeyMaxPic[nxt].dataIndex.map((i, index) => ({
title: finalUniKeyMaxPic[nxt].title,
dataIndex: i,
uniqueKey: nxt,
idx: index,
})),
);
}
});
filterColumns = filterColumns.concat(flatColumns);
// 对导出数据中图片的内容进行格式化
dataSource = dataSource.map(item => {
const newItem = {};
flatColumns.map(
(i: {
dataIndex: string;
title: string;
uniqueKey: string;
idx: number;
}) => {
newItem[i.dataIndex] = formatPictures(item[i.uniqueKey])?.[i.idx];
},
);
return {
...item,
...newItem,
};
});
return {
columns: filterColumns,
dataSource: dataSource,
};
}
}
return {
columns,
dataSource,
};
};
// 图片写入
const getBase64Image = img => {
let canvas = document.createElement('canvas');
canvas.width = img.width;
canvas.height = img.height;
let ctx = canvas.getContext('2d');
ctx?.drawImage(img, 0, 0, img.width, img.height);
let ext = img.src.substring(img.src.lastIndexOf('.') + 1).toLowerCase();
let dataURL = canvas.toDataURL('image/' + ext);
return dataURL;
};
const getBase64 = (url: string) => {
return new Promise((resolve, reject) => {
let image = new Image();
image.crossOrigin = 'anonymous';
image.src = url;
image.onload = () => resolve(getBase64Image(image));
image.onerror = error => reject(error);
});
};
async function addImage2Cell(workbook, worksheet, url, row, col) {
try {
const myBase64Image = await getBase64(url);
const imageId = workbook.addImage({
base64: myBase64Image,
extension: 'png',
});
worksheet.addImage(imageId, {
tl: { col: col, row: row },
br: { col: col + 1, row: row + 1 },
});
} catch (e) {
console.error('图片解析失败', url);
}
}
async function addImages2table({
workbook,
worksheet,
fields,
dataSource,
headerKeys,
offsetRow,
}) {
let picNumber = 0;
const colNums = fields.reduce(
(
cur: { index: number; dataIndex: string }[],
nxt: { uniqueKey?: string },
index: number,
) => {
if (Reflect.has(nxt, 'uniqueKey')) {
cur.push({
index: headerKeys.findIndex(
item => item === Reflect.get(nxt, 'dataIndex'),
),
dataIndex: Reflect.get(nxt, 'dataIndex') || '',
});
}
return cur;
},
[],
);
if (colNums?.length > 0) {
for (let i = 0; i < dataSource.length; i++) {
const promises: any[] = [];
for (let j = 0; j < colNums.length; j++) {
const pic = dataSource[i][colNums[j].dataIndex];
if (!!pic && picNumber < MAX_EXPORT_PICTURE) {
promises.push(
addImage2Cell(
workbook,
worksheet,
pic,
i + offsetRow,
colNums[j].index,
),
);
picNumber++;
}
}
await Promise.all(promises);
}
}
}性能优化
由于cell单元格的大小 = 数据量 * 列数
在数据量和列数同时增长的情况下，这个Excel列表将会无限变大。由于JS是单线程的，如何这么大的计算逻辑和渲染逻辑公用同一个线程的情况下，会直接造成浏览器卡死情况
web Worker
Web Worker 的作用，就是为 JavaScript 创造多线程环境，允许主线程创建 Worker 线程，将一些任务分配给后者运行。在主线程运行的同时，Worker 线程在后台运行，两者互不干扰。等到 Worker 线程完成计算任务，再把结果返回给主线程。这样的好处是，一些计算密集型或高延迟的任务，被 Worker 线程负担了，主线程（通常负责 UI 交互）就会很流畅，不会被阻塞或拖慢。
我们将数据获取+数据处理这些消耗CPU的逻辑全部转移到子线程中执行，不干扰主线程的渲染执行。
typescript
import * as Comlink from 'comlink';
import { saveAs } from 'file-saver';
export const openDownloadXLSXDialog = (url, saveName) => {
if (typeof url == 'object' && url instanceof Blob) {
url = URL.createObjectURL(url); // 创建blob地址
}
var aLink = document.createElement('a');
aLink.href = url;
aLink.download = saveName || ''; // HTML5新增的属性，指定保存文件名，可以不要后缀，注意，file:///模式下不会生效
var event;
if (window.MouseEvent) {
event = new MouseEvent('click');
} else {
event = document.createEvent('MouseEvents');
event.initMouseEvent(
'click',
true,
false,
window,
0,
0,
0,
0,
0,
false,
false,
false,
false,
0,
null,
);
}
aLink.dispatchEvent(event);
};
const str = `
importScripts("https://workorder-dz.oss-cn-zhangjiakou.aliyuncs.com/qy/workorder/default/comlink.js");
importScripts('https://workorder-dz.oss-cn-zhangjiakou.aliyuncs.com/pic/2023-10-18%2005:58:32_exceljs.min.js');
const DEFAULT_COLUMN_WIDTH = 20;
const DEFAULT_ROW_HEIGHT = 20;
function addHeaderStyle(row, attr) {
const { color, fontSize, horizontal, bold } = attr || {};
row.height = DEFAULT_ROW_HEIGHT;
row.eachCell((cell, colNumber) => {
cell.fill = {
type: 'pattern',
pattern: 'solid',
fgColor: { argb: color },
};
cell.font = {
bold: bold ?? true,
size: fontSize ?? 11,
name: '微软雅黑',
};
cell.alignment = {
vertical: 'middle',
wrapText: true,
horizontal: horizontal ?? 'left',
};
});
}
// 矩阵反转
function transposeMatrix(matrix) {
const numRows = matrix.length;
const numCols = matrix[0].length;
const result = [];
for (let col = 0; col < numCols; col++) {
const newRow = [];
for (let row = 0; row < numRows; row++) {
newRow.push(matrix[row][col]);
}
result.push(newRow);
}
return result;
}
function lastIndexOfObject(array, searchObject, fromIndex = array.length - 1) {
for (let i = fromIndex; i >= 0; i--) {
if (array[i] && typeof array[i] === 'object') {
const keysA = Object.keys(array[i]);
const keysB = Object.keys(searchObject);
if (keysA.length === keysB.length &&
keysA.every(key => array[i][key] === searchObject[key])) {
return i;
}
}
}
return -1;
}
function getExcelAlpha(index) {
let alpha = '';
let remainder;
while (index > 0) {
remainder = (index - 1) % 26;
alpha = String.fromCharCode(65 + remainder) + alpha;
index = Math.floor((index - 1) / 26);
}
return alpha;
}
const mergeColumnsCell = (
rowData,
worksheet,
rowNumber,
rowInstance,
) => {
// 当前列指针
let pointer = -1;
rowData.forEach((cellData, index) => {
if (index <= pointer) return;
pointer = lastIndexOfObject(rowData, cellData);
if (index !== pointer) {
const START = getExcelAlpha(index + 1);
const END = getExcelAlpha(pointer + 1);
const position = START + '' + (Number(rowNumber) + 1) + ':' + END + (Number(rowNumber) + 1)
worksheet.mergeCells(position);
const cell = rowInstance.getCell(index + 1);
cell.alignment = {
vertical: 'middle',
horizontal: 'center',
wrapText: true,
};
}
});
};
const mergeRowCells = (colData, worksheet, colNumber) => {
// 当前列指针
let pointer = -1;
colData.forEach((cellData, index) => {
if (index <= pointer)
return;
pointer = lastIndexOfObject(colData, cellData);
const COL = getExcelAlpha(colNumber + 1);
if (index !== pointer) {
const position = COL + '' + (Number(index) + 1) + ':' + COL + (Number(pointer) + 1);
worksheet.mergeCells(position);
}
});
};
function drawExcelHeader(worksheet, headerRows) {
// 行处理
headerRows === null || headerRows === void 0 ? void 0 : headerRows.map((row, index) => {
// 获取单行列头所有名称
const columnsName = row === null || row === void 0 ? void 0 : row.map(columns => columns.title);
// 单行列头插入
const rowHeader = worksheet.addRow(columnsName);
// 样式修改
addHeaderStyle(rowHeader, { color: 'dff8ff' });
// 行合并列头
mergeColumnsCell(row, worksheet, index, rowHeader);
});
if ((headerRows === null || headerRows === void 0 ? void 0 : headerRows.length) > 1) {
// 二维数组反转
const transList = transposeMatrix(headerRows);
transList === null || transList === void 0 ? void 0 : transList.map((row, index) => {
// 列处理
mergeRowCells(row, worksheet, index);
});
}
}
// 计算组件所有叶子节点个数
const getMaxChildLength = node => {
if (!node.children || node.children.length === 0) {
return 1; // 当前节点是叶子节点
}
let count = 0;
for (const childNode of node.children) {
count += getMaxChildLength(childNode);
}
return count;
};
const loopPushCell = (column, rowArray = [[]], index = 0, maxDepth = 0) => {
var _a, _b, _c;
if (index >= maxDepth)
return;
const count = getMaxChildLength(column);
const columns = (_b = (_a = new Array(count)) === null || _a === void 0 ? void 0 : _a.fill(null)) === null || _b === void 0 ? void 0 : _b.map(() => ({
title: column.title,
dataIndex: column.dataIndex,
}));
rowArray[index].push(...columns);
// 有子节点的时候，继续
if (Reflect.has(column, 'children')) {
++index;
(_c = column.children) === null || _c === void 0 ? void 0 : _c.map(item => loopPushCell(item, rowArray, index, maxDepth));
}
else {
// 无子节点的时候，子节点的数据继承父节点数据
loopPushCell(column, rowArray, ++index, maxDepth);
}
};
// 计算当前列头最大深度
function calculateMaxDepth(columns) {
let maxDepth = 1;
function calculateDepth(column, depth) {
if (column.children) {
for (const child of column.children) {
calculateDepth(child, depth + 1);
}
maxDepth = Math.max(maxDepth, depth + 1);
}
}
for (const column of columns) {
calculateDepth(column, 1);
}
return maxDepth;
}
function generateHeaders(columns, parentIndex) {
return columns === null || columns === void 0 ? void 0 : columns.map(col => {
const obj = {
// 显示的 name
header: col.title,
// 用于数据匹配的 key
key: col.dataIndex,
// 列宽
width: DEFAULT_COLUMN_WIDTH,
};
if (parentIndex) {
obj.parentKey = parentIndex;
}
if (Array.isArray(col.children)) {
obj.children = generateHeaders(col.children, col.dataIndex);
}
return obj;
});
}
function getColumnNumber(width) {
// 需要的列数，四舍五入
return Math.round(width / DEFAULT_COLUMN_WIDTH);
}
function mergeRowCell(headers, row, worksheet) {
// 当前列的索引
let colIndex = 1;
headers.forEach(header => {
const { width, children } = header;
if (children) {
children.forEach(child => {
colIndex += 1;
});
}
else {
// 需要的列数，四舍五入
const colNum = getColumnNumber(width);
// 如果 colNum > 1 说明需要合并
if (colNum > 1) {
worksheet.mergeCells(Number(row.number), colIndex, Number(row.number), colIndex + colNum - 1);
}
colIndex += colNum;
}
});
}
function addData2Table(dataSource, worksheet, headerKeys, headers) {
dataSource === null || dataSource === void 0 ? void 0 : dataSource.forEach((item) => {
const rowData = headerKeys === null || headerKeys === void 0 ? void 0 : headerKeys.map(key => item[key]);
const row = worksheet.addRow(rowData);
if (Reflect.has(item, 'exportIndex')) {
if (Number(item === null || item === void 0 ? void 0 : item.exportIndex) % 2 === 0) {
addHeaderStyle(row, { color: 'e8f3ff' });
}
}
mergeRowCell(headers, row, worksheet);
row.height = 26;
// 设置行样式, wrapText: 自动换行
row.alignment = { vertical: 'middle', wrapText: true, shrinkToFit: false };
row.font = { size: 11, name: '微软雅黑' };
});
}
const getWorkBook = ({ columns: fields, dataSource, fileName }) => {
const workbook = new ExcelJS.Workbook();
const worksheet = workbook.addWorksheet(fileName);
// 设置 sheet 的默认行高
worksheet.properties.defaultRowHeight = 20;
const headers = generateHeaders(fields);
// 计算当前Excel列头最大深度
const maxDepth = calculateMaxDepth(fields);
const excelHeaderRow = new Array(maxDepth).fill(null).map(() => []);
fields?.map(item => loopPushCell(item, excelHeaderRow, 0, maxDepth));
drawExcelHeader(worksheet, excelHeaderRow);
addData2Table(
dataSource,
worksheet,
excelHeaderRow?.[excelHeaderRow?.length - 1]?.map(
(item) => item?.dataIndex,
),
headers,
);
console.log('xxx1', worksheet.columns, DEFAULT_COLUMN_WIDTH)
// 给每列设置固定宽度
worksheet.columns = worksheet.columns.map(col => ({
...col,
width: DEFAULT_COLUMN_WIDTH,
}));
return workbook.xlsx.writeBuffer()
};
Comlink.expose(getWorkBook);
`;
const blob = new Blob([str]);
const url = window.URL.createObjectURL(blob);
const worker = new Worker(url);
export const exportExcel = Comlink.wrap(worker);
type exportExcelDefaultType = {
columns: any[];
dataSource?: any[];
fileName: string;
};
// 默认导出表格中的所有数据
export const exportExcelDefault = async ({
columns = [],
dataSource = [],
fileName,
}: exportExcelDefaultType) => {
// @ts-ignore
const blobExcel = await exportExcel({
columns,
dataSource,
fileName,
});
const blob = new Blob([blobExcel], { type: '' });
saveAs(blob, `${fileName}.xlsx`);
};web Worker里面没有办法操作window和document对象，所以针对需要生成图片的场景，没有办法移入到web worker中执行，需要取舍。所以这里在不导出图片的时候走子线程下载。在导出的时候可以让用户自己抉择是否需要下载图片