Day 52 神经网络调参指南

@浙大疏锦行

推荐的调参顺序

1. 设置随机种子
2. 选择合适的权重初始化
3. 调整学习率和调度器
4. 添加正则化
5. 配置早停

import torch
import torch.nn as nn

# 定义一个简单的线性模型
# 输入2个维度，输出1个维度
class SimpleNet(nn.Module):
    def __init__(self):
        super(SimpleNet, self).__init__()
        # 线性层：y = w1*x1 + w2*x2 + b
        self.linear = nn.Linear(2, 1)
        
    def forward(self, x):
        return self.linear(x)

# 创建模型实例（每次运行权重不同）
model = SimpleNet()

# 查看模型参数
print("模型参数（未设置种子，每次运行可能不同）:")
for name, param in model.named_parameters():
    print(f"{name}: {param.data}")

import torch
import torch.nn as nn

# 演示梯度消失问题
def demo_gradient_problem():
    """展示权重初始化对梯度的影响"""
    
    # 定义深层网络
    class DeepNet(nn.Module):
        def __init__(self, init_scale):
            super().__init__()
            layers = []
            for _ in range(10):  # 10层网络
                linear = nn.Linear(100, 100, bias=False)
                # 使用指定的初始化尺度
                nn.init.normal_(linear.weight, mean=0, std=init_scale)
                layers.append(linear)
                layers.append(nn.Tanh())
            self.net = nn.Sequential(*layers)
            
        def forward(self, x):
            return self.net(x)
    
    # 测试不同初始化尺度
    for scale in [0.01, 0.1, 1.0]:
        model = DeepNet(scale)
        x = torch.randn(1, 100)
        output = model(x)
        print(f"初始化std={scale}: 输出均值={output.mean().item():.6f}, 输出std={output.std().item():.6f}")

demo_gradient_problem()

import torch
import torch.nn as nn
import torch.optim as optim
import matplotlib.pyplot as plt

# 演示不同学习率调度器
def demo_lr_schedulers():
    """展示不同学习率调度器的效果"""
    
    epochs = 100
    initial_lr = 0.1
    
    schedulers = {
        'StepLR': lambda opt: optim.lr_scheduler.StepLR(opt, step_size=30, gamma=0.1),
        'ExponentialLR': lambda opt: optim.lr_scheduler.ExponentialLR(opt, gamma=0.95),
        'CosineAnnealingLR': lambda opt: optim.lr_scheduler.CosineAnnealingLR(opt, T_max=100),
        'ReduceLROnPlateau': lambda opt: optim.lr_scheduler.ReduceLROnPlateau(opt, mode='min', patience=10)
    }
    
    fig, axes = plt.subplots(2, 2, figsize=(12, 8))
    axes = axes.flatten()
    
    for idx, (name, scheduler_fn) in enumerate(schedulers.items()):
        # 创建简单模型和优化器
        model = nn.Linear(10, 1)
        optimizer = optim.SGD(model.parameters(), lr=initial_lr)
        scheduler = scheduler_fn(optimizer)
        
        lrs = []
        for epoch in range(epochs):
            lrs.append(optimizer.param_groups[0]['lr'])
            if name == 'ReduceLROnPlateau':
                scheduler.step(1.0)  # 需要传入loss值
            else:
                scheduler.step()
        
        axes[idx].plot(lrs)
        axes[idx].set_title(name)
        axes[idx].set_xlabel('Epoch')
        axes[idx].set_ylabel('Learning Rate')
        axes[idx].grid(True)
    
    plt.tight_layout()
    plt.show()

demo_lr_schedulers()

import torch.optim as optim

# L2正则化（权重衰减）在PyTorch中通过weight_decay参数实现
model = nn.Linear(100, 10)

# 创建带L2正则化的优化器
optimizer = optim.Adam(
    model.parameters(), 
    lr=0.001, 
    weight_decay=1e-4  # L2正则化系数
)

print(f"优化器配置: {optimizer}")
print(f"L2正则化系数: {optimizer.param_groups[0]['weight_decay']}")

class EarlyStopping:
    """
    早停类：监控验证集性能，在性能不再提升时停止训练
    """
    def __init__(self, patience=7, min_delta=0, verbose=True):
        """
        Args:
            patience: 允许多少个epoch性能不提升
            min_delta: 性能提升的最小阈值
            verbose: 是否打印信息
        """
        self.patience = patience
        self.min_delta = min_delta
        self.verbose = verbose
        self.counter = 0
        self.best_score = None
        self.early_stop = False
        self.best_model_state = None
        
    def __call__(self, val_loss, model):
        score = -val_loss  # 转换为越大越好
        
        if self.best_score is None:
            # 第一次调用
            self.best_score = score
            self.save_checkpoint(model)
        elif score < self.best_score + self.min_delta:
            # 性能没有提升
            self.counter += 1
            if self.verbose:
                print(f'EarlyStopping counter: {self.counter}/{self.patience}')
            if self.counter >= self.patience:
                self.early_stop = True
        else:
            # 性能提升
            self.best_score = score
            self.save_checkpoint(model)
            self.counter = 0
            
    def save_checkpoint(self, model):
        """保存模型状态"""
        self.best_model_state = model.state_dict().copy()
        if self.verbose:
            print(f'验证损失下降，保存模型...')
            
    def load_best_model(self, model):
        """加载最佳模型"""
        model.load_state_dict(self.best_model_state)
        return model

# 使用示例
print("早停策略使用示例:")
print("""
early_stopping = EarlyStopping(patience=7)

for epoch in range(epochs):
    train_loss = train(model, train_loader)
    val_loss = validate(model, val_loader)
    
    early_stopping(val_loss, model)
    
    if early_stopping.early_stop:
        print("Early stopping triggered!")
        break

# 加载最佳模型
model = early_stopping.load_best_model(model)
""")