Transformer时间序列预测：从Informer到TimesFM的完整演进
一、引言
Transformer 在 NLP 取得巨大成功后，时间序列预测领域也经历了一场"Transformer 革命"。从 Informer 解决长序列效率问题，到 Autoformer 引入频域分解，再到 Google 的 TimesFM 基础模型，Transformer 时序预测正走向大一统。
本文将覆盖从原理到实战的完整路线：Informer→Autoformer→PatchTST→TimesFM。
二、为什么Transformer适合时间序列
| 传统方法 | 局限 | Transformer优势 |
|---|---|---|
| ARIMA/ETS | 仅线性，单变量 | 非线性，多变量 |
| LSTM/GRU | 长序列梯度消失 | 自注意力全局建模 |
| Prophet | 需人工特征 | 自动学习模式 |
| XGBoost | 需滚动窗口 | 直接映射序列到序列 |
三、Informer：高效长序列预测
3.1 ProbSparse注意力
python
import torch
import torch.nn as nn
import numpy as np
class ProbSparseAttention(nn.Module):
"""Informer核心：概率稀疏自注意力 O(L·logL)"""
def __init__(self, d_model=512, n_heads=8, factor=5):
super().__init__()
self.d_model = d_model
self.n_heads = n_heads
self.factor = factor # 采样因子
self.d_k = d_model // n_heads
self.W_q = nn.Linear(d_model, d_model)
self.W_k = nn.Linear(d_model, d_model)
self.W_v = nn.Linear(d_model, d_model)
self.W_o = nn.Linear(d_model, d_model)
def _prob_QK(self, Q, K, sample_k, n_top):
"""概率稀疏度量 M(q_i, K) = max_j{q_i·k_j^T/√d} - mean_j{q_i·k_j^T/√d}"""
B, H, L_Q, D = Q.shape
_, _, L_K, _ = K.shape
# 随机采样 K 的一部分来计算稀疏性度量
index_sample = torch.randint(L_K, (L_Q, sample_k))
K_sample = K[:, :, index_sample, :] # [B,H,L_Q,sample_k,D]
# 计算 Q-K 点积（采样部分）
Q_K_sample = torch.matmul(
Q.unsqueeze(-2), # [B,H,L_Q,1,D]
K_sample.transpose(-2, -1) # [B,H,L_Q,D,sample_k]
).squeeze(-2) / np.sqrt(D) # [B,H,L_Q,sample_k]
# M度量 = max - mean
M = Q_K_sample.max(dim=-1)[0] - Q_K_sample.mean(dim=-1)
# 选择 top-u 个查询
M_top = M.topk(n_top, sorted=False)[1] # [B,H,n_top]
return M_top
def forward(self, queries, keys, values):
B, L, _ = queries.shape
H, D = self.n_heads, self.d_k
# 线性投影
Q = self.W_q(queries).view(B, -1, H, D).transpose(1, 2)
K = self.W_k(keys).view(B, -1, H, D).transpose(1, 2)
V = self.W_v(values).view(B, -1, H, D).transpose(1, 2)
# 1. 计算稀疏性度量，选择 top-u
u = self.factor * int(np.log(L))
M_top = self._prob_QK(Q, K, sample_k=int(np.log(L)), n_top=u)
# 2. 只计算 top-u 个查询的完整注意力
Q_reduce = Q.gather(2, M_top.unsqueeze(-1).expand(-1, -1, -1, D))
# 标准注意力计算（只对top-u查询）
attn = torch.matmul(Q_reduce, K.transpose(-2, -1)) / np.sqrt(D)
attn = torch.softmax(attn, dim=-1)
# 3. 对 top-u 查询计算输出
context = torch.matmul(attn, V) # [B,H,u,D]
# 4. 填充：非top-u查询用V的均值替代
V_mean = V.mean(dim=2).unsqueeze(2).expand(-1, -1, L - u, -1)
out = torch.cat([context, V_mean], dim=2)
return self.W_o(out.transpose(1, 2).contiguous().view(B, L, -1))3.2 蒸馏层
python
class DistillingLayer(nn.Module):
"""自注意力蒸馏：Conv1d + MaxPool → 减半序列长度"""
def __init__(self, d_model):
super().__init__()
self.conv = nn.Conv1d(d_model, d_model, 3, padding=1)
self.norm = nn.BatchNorm1d(d_model)
self.pool = nn.MaxPool1d(3, stride=2, padding=1)
def forward(self, x):
# x: [B, L, D] → [B, D, L]
x = x.transpose(1, 2)
x = self.conv(x)
x = self.norm(x)
x = torch.relu(x)
x = self.pool(x) # 长度减半
return x.transpose(1, 2)四、Autoformer：频域分解
4.1 序列分解模块
python
class SeriesDecomposition(nn.Module):
"""移动平均分解：序列 → 趋势项 + 季节项"""
def __init__(self, kernel_size=25):
super().__init__()
self.moving_avg = nn.AvgPool1d(
kernel_size=kernel_size,
stride=1,
padding=kernel_size // 2
)
def forward(self, x):
"""x: [B, L, D]"""
# 趋势项：移动平均
trend = self.moving_avg(x.transpose(1, 2)).transpose(1, 2)
# 季节项：原始 - 趋势
seasonal = x - trend
return seasonal, trend4.2 Auto-Correlation 频域注意力
python
class AutoCorrelation(nn.Module):
"""自相关机制：用FFT高效计算周期相似性"""
def __init__(self, d_model=512, n_heads=8, top_k=3):
super().__init__()
self.n_heads = n_heads
self.d_k = d_model // n_heads
self.top_k = top_k # 保留最强的k个周期
self.W_qkv = nn.Linear(d_model, d_model * 3)
self.W_o = nn.Linear(d_model, d_model)
def time_delay_agg(self, values, corr_indices):
"""基于最强周期的时延聚合"""
B, H, L, D = values.shape
top_k = corr_indices.shape[-1]
# 按索引 gather + Roll
aligned_values = []
for k in range(top_k):
idx = corr_indices[:, :, :, k] # [B,H,L]
# Roll操作：将时间序列按 idx 移位
v_k = torch.gather(values, 2,
idx.unsqueeze(-1).expand(-1, -1, -1, D))
aligned_values.append(v_k)
# 拼接并投影
aligned = torch.stack(aligned_values, dim=-1) # [B,H,L,D,top_k]
return aligned.mean(dim=-1)
def forward(self, x):
B, L, D_in = x.shape
H = self.n_heads
# QKV投影
qkv = self.W_qkv(x).reshape(B, L, 3, H, self.d_k).permute(2, 0, 3, 1, 4)
Q, K, V = qkv[0], qkv[1], qkv[2]
# 频域自相关: R(τ) = FFT^{-1}(S_xy(f))
# S_xy(f) = FFT(Q)_f · conj(FFT(K)_f)
Q_fft = torch.fft.rfft(Q, dim=2)
K_fft = torch.fft.rfft(K, dim=2)
# 频域相乘
S = Q_fft * torch.conj(K_fft)
# 逆FFT得到时域自相关
corr = torch.fft.irfft(S, dim=2) # [B,H,L]
# 选择最强的 top_k 个延迟
top_corr, top_indices = torch.topk(corr, self.top_k, dim=2)
# 时延聚合
out = self.time_delay_agg(V, top_indices)
return self.W_o(out.transpose(1, 2).reshape(B, L, D_in))五、PatchTST：分块革命
python
class PatchTST(nn.Module):
"""将时间序列分割为重叠patches，用ViT风格处理"""
def __init__(self, patch_len=16, stride=8, d_model=128, n_layers=3):
super().__init__()
self.patch_len = patch_len
self.stride = stride
# Patch嵌入: 每个patch → d_model维
self.patch_embed = nn.Linear(patch_len, d_model)
# 可学习位置编码
self.pos_embed = nn.Parameter(torch.randn(1, 512, d_model))
# Transformer编码器
self.encoder = nn.TransformerEncoder(
nn.TransformerEncoderLayer(d_model, nhead=8, batch_first=True),
num_layers=n_layers
)
# 输出头
self.head = nn.Linear(d_model, patch_len)
def forward(self, x):
"""x: [B, L, V] L=序列长度, V=变量数"""
B, L, V = x.shape
# 1. 分块 (Patching)
patches = x.unfold(dimension=1, size=self.patch_len, step=self.stride)
# [B, num_patches, V, patch_len]
num_patches = patches.shape[1]
# 2. 独立通道策略（每个变量独立处理）
patches = patches.permute(0, 2, 1, 3) # [B, V, num_patches, patch_len]
patches = patches.reshape(B * V, num_patches, self.patch_len)
# 3. Patch嵌入
embeds = self.patch_embed(patches) # [B*V, num_patches, d_model]
embeds = embeds + self.pos_embed[:, :num_patches, :]
# 4. Transformer编码
encoded = self.encoder(embeds)
# 5. 输出预测
pred = self.head(encoded) # [B*V, num_patches, patch_len]
# 6. 重组
pred = pred.reshape(B, V, num_patches, self.patch_len)
pred = pred.permute(0, 2, 1, 3) # [B, num_patches, V, patch_len]
# 用最后一个patch的预测作为最终输出
return pred[:, -1, :, :] # [B, V, patch_len]六、TimesFM：时序基础模型
python
import timesfm
# 加载预训练 TimesFM 模型
model = timesfm.TimesFm(
context_len=512, # 历史窗口
horizon_len=128, # 预测长度
input_patch_len=32,
output_patch_len=128,
num_layers=20,
model_dims=1280,
)
# 加载权重
model.load_from_checkpoint("google/timesfm-1.0-200m")
# 零样本预测
forecast = model.forecast(
historical_values,
horizon=128
)
print(f"预测: {forecast.shape}")
# 与传统方法对比
# TimesFM优势:
# 1. 预训练：在1000亿+时间点上训练
# 2. 零样本：无需微调即可预测新序列
# 3. 上下文学习：类似LLM的In-Context Learning
# 4. 多频率：日/周/月/年频率无缝切换七、异常检测
python
class AnomalyDetector(nn.Module):
"""基于预测误差的时间序列异常检测"""
def __init__(self, predictor, threshold_percentile=95):
self.predictor = predictor
self.threshold = threshold_percentile
def detect(self, sequence, window=96, horizon=24):
"""滑动窗口预测 → 计算误差 → 标记异常"""
anomalies = []
reconstruction_errors = []
for i in range(0, len(sequence) - window - horizon):
# 历史窗口
hist = sequence[i:i+window]
# 真实未来值
true_future = sequence[i+window:i+window+horizon]
# 预测
pred = self.predictor(hist.unsqueeze(0))
# 重建误差
error = torch.abs(pred - true_future).mean(dim=-1)
reconstruction_errors.append(error.item())
# 阈值判断
if error > self.error_threshold:
anomalies.append(i + window)
return anomalies, reconstruction_errors
def fit_threshold(self, train_errors):
"""从训练数据学习异常阈值"""
self.error_threshold = np.percentile(
train_errors, self.threshold_percentile
)八、总结
| 模型 | 核心创新 | 适合场景 |
|---|---|---|
| Informer | ProbSparse注意力+蒸馏 | 极长序列(1000+) |
| Autoformer | FFT自相关+趋势分解 | 强周期序列 |
| PatchTST | 分块+独立通道 | 多变量通用预测 |
| TimesFM | 预训练基础模型 | 零样本快速预测 |
选择建议：数据充足训练PatchTST，长序列用Informer，零样本用TimesFM。