强化学习实战：从 DQN 到 PPO 的完整指南
1. 引言
强化学习（RL）是让智能体通过与环境交互来学习最优策略的方法。从 Atari 游戏到机器人控制，从 RLHF 到代码生成，RL 的应用越来越广泛。本文将从基础概念到前沿算法，系统讲解强化学习。
核心概念：
智能体 (Agent) ←→ 环境 (Environment)
- 状态 (State): s_t
- 动作 (Action): a_t
- 奖励 (Reward): r_t
- 策略 (Policy): π(a|s)
- 价值函数 (Value): V(s) 或 Q(s,a)
目标：最大化累积奖励 E[Σ γ^t · r_t]2. DQN（Deep Q-Network）
2.1 原理
Q-learning 更新规则：
Q(s,a) ← Q(s,a) + α[r + γ·max Q(s',a') - Q(s,a)]
DQN 改进：
1. 用神经网络近似 Q 函数
2. 经验回放（Experience Replay）
3. 目标网络（Target Network）2.2 DQN 实现
python
import torch
import torch.nn as nn
import numpy as np
from collections import deque
import random
class DQN(nn.Module):
"""DQN 网络"""
def __init__(self, state_dim, action_dim, hidden=256):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, hidden), nn.ReLU(),
nn.Linear(hidden, hidden), nn.ReLU(),
nn.Linear(hidden, action_dim),
)
def forward(self, x):
return self.net(x)
class ReplayBuffer:
"""经验回放缓冲区"""
def __init__(self, capacity=100000):
self.buffer = deque(maxlen=capacity)
def push(self, state, action, reward, next_state, done):
self.buffer.append((state, action, reward, next_state, done))
def sample(self, batch_size):
batch = random.sample(self.buffer, batch_size)
states, actions, rewards, next_states, dones = zip(*batch)
return (
torch.FloatTensor(np.array(states)),
torch.LongTensor(actions),
torch.FloatTensor(rewards),
torch.FloatTensor(np.array(next_states)),
torch.FloatTensor(dones),
)
def __len__(self):
return len(self.buffer)
class DQNAgent:
"""DQN 智能体"""
def __init__(self, state_dim, action_dim, lr=1e-3, gamma=0.99,
epsilon_start=1.0, epsilon_end=0.01, epsilon_decay=0.995):
self.action_dim = action_dim
self.gamma = gamma
self.epsilon = epsilon_start
self.epsilon_end = epsilon_end
self.epsilon_decay = epsilon_decay
# Q 网络和目标网络
self.q_net = DQN(state_dim, action_dim)
self.target_net = DQN(state_dim, action_dim)
self.target_net.load_state_dict(self.q_net.state_dict())
self.optimizer = torch.optim.Adam(self.q_net.parameters(), lr=lr)
self.buffer = ReplayBuffer()
def select_action(self, state):
"""ε-贪婪策略"""
if random.random() < self.epsilon:
return random.randint(0, self.action_dim - 1)
with torch.no_grad():
state_t = torch.FloatTensor(state).unsqueeze(0)
q_values = self.q_net(state_t)
return q_values.argmax(dim=1).item()
def train_step(self, batch_size=64):
"""训练一步"""
if len(self.buffer) < batch_size:
return
states, actions, rewards, next_states, dones = self.buffer.sample(batch_size)
# 当前 Q 值
q_values = self.q_net(states).gather(1, actions.unsqueeze(1)).squeeze(1)
# 目标 Q 值
with torch.no_grad():
next_q = self.target_net(next_states).max(dim=1)[0]
target_q = rewards + self.gamma * next_q * (1 - dones)
# 更新
loss = nn.MSELoss()(q_values, target_q)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
return loss.item()
def update_target(self):
"""更新目标网络"""
self.target_net.load_state_dict(self.q_net.state_dict())
def decay_epsilon(self):
"""衰减探索率"""
self.epsilon = max(self.epsilon_end, self.epsilon * self.epsilon_decay)2.3 训练循环
python
import gymnasium as gym
def train_dqn(env_name="CartPole-v1", episodes=500):
env = gym.make(env_name)
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
agent = DQNAgent(state_dim, action_dim)
rewards_history = []
for episode in range(episodes):
state, _ = env.reset()
total_reward = 0
while True:
action = agent.select_action(state)
next_state, reward, terminated, truncated, _ = env.step(action)
done = terminated or truncated
agent.buffer.push(state, action, reward, next_state, float(done))
agent.train_step()
state = next_state
total_reward += reward
if done:
break
agent.decay_epsilon()
if episode % 10 == 0:
agent.update_target()
rewards_history.append(total_reward)
if episode % 50 == 0:
avg = np.mean(rewards_history[-50:])
print(f"Episode {episode}, Avg Reward: {avg:.1f}, ε: {agent.epsilon:.3f}")
return agent3. 策略梯度（Policy Gradient）
3.1 REINFORCE 算法
python
class PolicyNetwork(nn.Module):
"""策略网络"""
def __init__(self, state_dim, action_dim, hidden=256):
super().__init__()
self.net = nn.Sequential(
nn.Linear(state_dim, hidden), nn.ReLU(),
nn.Linear(hidden, hidden), nn.ReLU(),
nn.Linear(hidden, action_dim), nn.Softmax(dim=-1),
)
def forward(self, x):
return self.net(x)
def train_reinforce(env_name="CartPole-v1", episodes=1000, lr=1e-3, gamma=0.99):
env = gym.make(env_name)
state_dim = env.observation_space.shape[0]
action_dim = env.action_space.n
policy = PolicyNetwork(state_dim, action_dim)
optimizer = torch.optim.Adam(policy.parameters(), lr=lr)
for episode in range(episodes):
states, actions, rewards = [], [], []
state, _ = env.reset()
while True:
state_t = torch.FloatTensor(state).unsqueeze(0)
probs = policy(state_t)
action = torch.multinomial(probs, 1).item()
next_state, reward, terminated, truncated, _ = env.step(action)
done = terminated or truncated
states.append(state)
actions.append(action)
rewards.append(reward)
state = next_state
if done:
break
# 计算折扣回报
returns = []
G = 0
for r in reversed(rewards):
G = r + gamma * G
returns.insert(0, G)
returns = torch.FloatTensor(returns)
returns = (returns - returns.mean()) / (returns.std() + 1e-8)
# 策略梯度更新
states_t = torch.FloatTensor(np.array(states))
actions_t = torch.LongTensor(actions)
probs = policy(states_t)
log_probs = torch.log(probs.gather(1, actions_t.unsqueeze(1)).squeeze(1))
loss = -(log_probs * returns).mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
if episode % 50 == 0:
print(f"Episode {episode}, Total Reward: {sum(rewards):.1f}")4. PPO（Proximal Policy Optimization）
4.1 核心思想
PPO 通过裁剪目标函数限制策略更新幅度：
L_CLIP = E[min(r_t(θ)·A_t, clip(r_t(θ), 1-ε, 1+ε)·A_t)]
其中：
r_t(θ) = π_θ(a|s) / π_θ_old(a|s) （概率比）
A_t = 优势函数估计
ε = 裁剪参数（通常 0.1-0.2）4.2 PPO 实现
python
class PPO:
"""PPO 算法"""
def __init__(self, state_dim, action_dim, lr=3e-4, gamma=0.99,
gae_lambda=0.95, clip_epsilon=0.2, epochs=10):
self.gamma = gamma
self.gae_lambda = gae_lambda
self.clip_epsilon = clip_epsilon
self.epochs = epochs
# Actor-Critic 网络
self.actor = PolicyNetwork(state_dim, action_dim)
self.critic = nn.Sequential(
nn.Linear(state_dim, 256), nn.ReLU(),
nn.Linear(256, 256), nn.ReLU(),
nn.Linear(256, 1),
)
self.optimizer = torch.optim.Adam(
list(self.actor.parameters()) + list(self.critic.parameters()),
lr=lr
)
def compute_gae(self, rewards, values, dones):
"""计算 GAE（广义优势估计）"""
advantages = []
gae = 0
for t in reversed(range(len(rewards))):
if t == len(rewards) - 1:
next_value = 0
else:
next_value = values[t + 1]
delta = rewards[t] + self.gamma * next_value * (1 - dones[t]) - values[t]
gae = delta + self.gamma * self.gae_lambda * (1 - dones[t]) * gae
advantages.insert(0, gae)
returns = [adv + val for adv, val in zip(advantages, values)]
return torch.FloatTensor(advantages), torch.FloatTensor(returns)
def update(self, trajectories):
"""PPO 更新"""
states = torch.FloatTensor(np.array(trajectories["states"]))
actions = torch.LongTensor(trajectories["actions"])
old_log_probs = torch.FloatTensor(trajectories["log_probs"])
rewards = trajectories["rewards"]
values = trajectories["values"]
dones = trajectories["dones"]
advantages, returns = self.compute_gae(rewards, values, dones)
advantages = (advantages - advantages.mean()) / (advantages.std() + 1e-8)
for _ in range(self.epochs):
# 当前策略
probs = self.actor(states)
log_probs = torch.log(probs.gather(1, actions.unsqueeze(1)).squeeze(1))
current_values = self.critic(states).squeeze(1)
# 概率比
ratio = torch.exp(log_probs - old_log_probs)
# 裁剪目标
surr1 = ratio * advantages
surr2 = torch.clamp(ratio, 1 - self.clip_epsilon, 1 + self.clip_epsilon) * advantages
actor_loss = -torch.min(surr1, surr2).mean()
# Critic 损失
critic_loss = nn.MSELoss()(current_values, returns)
# 熵正则化
entropy = -(probs * torch.log(probs + 1e-8)).sum(dim=-1).mean()
# 总损失
loss = actor_loss + 0.5 * critic_loss - 0.01 * entropy
self.optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(
list(self.actor.parameters()) + list(self.critic.parameters()),
0.5
)
self.optimizer.step()5. 强化学习在 LLM 中的应用
python
# PPO 在 RLHF 中的应用（简化版）
from trl import PPOTrainer, PPOConfig
config = PPOConfig(
learning_rate=1.41e-5,
batch_size=64,
ppo_epochs=4,
kl_penalty="kl",
init_kl_coef=0.2,
)
# 奖励来自人类偏好训练的奖励模型
# 策略是 LLM 本身
# 状态是 prompt，动作是生成的 token6. 算法对比
| 算法 | 类型 | 动作空间 | 样本效率 | 稳定性 |
|---|---|---|---|---|
| DQN | 值函数 | 离散 | 高 | 中 |
| REINFORCE | 策略梯度 | 连续/离散 | 低 | 低 |
| A2C | Actor-Critic | 连续/离散 | 中 | 中 |
| PPO | Actor-Critic | 连续/离散 | 高 | 高 |
| SAC | Actor-Critic | 连续 | 高 | 高 |
7. 总结
强化学习的核心算法：
- DQN：值函数方法，适合离散动作空间
- 策略梯度：直接优化策略，适合连续动作
- PPO：当前最流行的通用 RL 算法，稳定高效
- RLHF：PPO 在 LLM 对齐中的成功应用