4.权重衰减(weight decay)

4.1 手动实现权重衰减

python 复制代码
import torch
from torch import nn
from torch.utils.data import TensorDataset,DataLoader
import matplotlib.pyplot as plt
def synthetic_data(w,b,num_inputs):
    X=torch.normal(0,1,size=(num_inputs,w.shape[0]))
    y=X@w+b
    y+=torch.normal(0,0.1,size=y.shape)
    return X,y
def load_array(data,batch_size,is_train=True):
    dataset=TensorDataset(*data)
    return DataLoader(dataset,batch_size=batch_size,shuffle=is_train)
def init_params(num_inputs):
    w=torch.normal(0,1,size=(num_inputs,1),requires_grad=True)
    b=torch.zeros(1,requires_grad=True)
    return [w,b]
def l2_penalty(w):
    return 0.5*torch.sum(w.pow(2))

def linear_reg(X,w,b):
    return torch.matmul(X,w)+b
def mse_loss(y_hat,y):
    return (y_hat-y)**2/2
def sgd(params,lr,batch_size):
    for params in params:
        params.data-=lr*params.grad/batch_size
        params.grad.zero_()
def evaluate_loss(net, data_iter, loss):
    total_loss, total_samples = 0.0, 0
    for X, y in data_iter:
        l = loss(net(X), y)
        total_loss += l.sum().item()
        total_samples += y.numel()
    return total_loss / total_samples
n_train,n_test,num_inputs,batch_size=20,100,200,5
true_w,true_b=torch.ones((num_inputs,1))*0.01,0.05
train_data=synthetic_data(true_w,true_b,n_train)
test_data=synthetic_data(true_w,true_b,n_test)
train_iter=load_array(train_data,batch_size)
test_iter=load_array(test_data,batch_size,is_train=False)
w,b=init_params(num_inputs)
net=lambda X:linear_reg(X,w,b)
loss=mse_loss
num_epochs,lr,lambd=10,0.05,3
#animator=SimpleAnimator()
for epoch in range(num_epochs):
    for X,y in train_iter:
        l=loss(net(X),y)+lambd*l2_penalty(w)
        l.sum().backward()
        sgd([w,b],lr,batch_size)
    if (epoch+1)%5==0:
        train_loss=evaluate_loss(net,train_iter,loss)
        test_loss=evaluate_loss(net,test_iter,loss)
        #animator.add(epoch+1,train_loss,test_loss)
        print(f"Epoch {epoch+1}: Train Loss: {train_loss:.4f},test Loss: {test_loss:.4f}")
print('w的L2范数是:', torch.norm(w).item())
plt.show()

4.2 简单实现权重衰减

python 复制代码
import torch
from torch import nn
from torch.utils.data import TensorDataset,DataLoader
import matplotlib.pyplot as plt
def synthetic_data(w,b,num_inputs):
    X=torch.normal(0,1,size=(num_inputs,w.shape[0]))
    y=X@w+b
    y+=torch.normal(0,0.1,size=y.shape)
    return X,y
def load_array(data,batch_size,is_train=True):
    dataset=TensorDataset(*data)
    return DataLoader(dataset,batch_size=batch_size,shuffle=is_train)
def init_params(num_inputs):
    w=torch.normal(0,1,size=(num_inputs,1),requires_grad=True)
    b=torch.zeros(1,requires_grad=True)
    return [w,b]
def l2_penalty(w):
    return 0.5*torch.sum(w.pow(2))
def linear_reg(X,w,b):
    return torch.matmul(X,w)+b
def mse_loss(y_hat,y):
    return ((y_hat-y)**2).sum()/2
def evaluate_loss(net, data_iter, loss):
    total_loss, total_samples = 0.0, 0
    for X, y in data_iter:
        l = loss(net(X), y)
        total_loss += l.item()*y.shape[0]
        total_samples += y.numel()
    return total_loss / total_samples
n_train,n_test,num_inputs,batch_size=20,100,200,5
true_w,true_b=torch.ones((num_inputs,1))*0.01,0.05
train_data=synthetic_data(true_w,true_b,n_train)
test_data=synthetic_data(true_w,true_b,n_test)
train_iter=load_array(train_data,batch_size)
test_iter=load_array(test_data,batch_size,is_train=False)
w,b=init_params(num_inputs)
net=lambda X:linear_reg(X,w,b)
loss=mse_loss
num_epochs,lr,lambd=100,0.001,3
optimizer=torch.optim.SGD([w,b],lr=lr,weight_decay=0.001)
#animator=SimpleAnimator()
for epoch in range(num_epochs):
    for X,y in train_iter:
        optimizer.zero_grad()
        l=loss(net(X),y)
        l.backward()
        #sgd([w,b],lr,batch_size)
        optimizer.step() 
    if (epoch+1)%5==0:
        train_loss=evaluate_loss(net,train_iter,loss)
        test_loss=evaluate_loss(net,test_iter,loss)
        #animator.add(epoch+1,train_loss,test_loss)
        print(f"Epoch {epoch+1}: Train Loss: {train_loss:.4f},test Loss: {test_loss:.4f}")
print('w的L2范数是:', torch.norm(w).item())
plt.show()
相关推荐
2601_9679801130 分钟前
python pytest怎样测试代码
python·pytest·测试·文档·断言
伞伞悦读1 小时前
【第92期】逻辑回归的分数概率与阈值的实验与排查
人工智能·python·深度学习·机器学习
ch6231 小时前
🚀 从零构建本地文档智能审查 Agent:Vibe Coding 实战指南
python
摇滚侠2 小时前
《On Java 中文版 基础卷》阅读笔记 接口 18
java·笔记·python
电科_银尘2 小时前
【大模型算法与工程 27】训练配方:批量、学习率、精度与梯度裁剪
学习·算法·机器学习
Thanks_ks3 小时前
AI Agent 开发入门:用 Function Calling 让大模型学会 “动手干活”,附订单查询机器人实战
python·ai·大模型·openai·agent·通义千问·mcp
专注仿真3 小时前
智能体训练之模仿学习:GAIL 算法详解与实战
python·算法·仿真·多智能体强化学习·afsim智能体训练
databook4 小时前
处理一批 PDF,怎么算真的处理完了
后端·python
承渊政道6 小时前
【Python编程快速上手—让繁琐工作自动化】(列表:把数据排成队,从下标取值到深拷贝)
开发语言·python·pycharm·自动化·列表