4.权重衰减(weight decay)

4.1 手动实现权重衰减

import torch
from torch import nn
from torch.utils.data import TensorDataset,DataLoader
import matplotlib.pyplot as plt
def synthetic_data(w,b,num_inputs):
    X=torch.normal(0,1,size=(num_inputs,w.shape[0]))
    y=X@w+b
    y+=torch.normal(0,0.1,size=y.shape)
    return X,y
def load_array(data,batch_size,is_train=True):
    dataset=TensorDataset(*data)
    return DataLoader(dataset,batch_size=batch_size,shuffle=is_train)
def init_params(num_inputs):
    w=torch.normal(0,1,size=(num_inputs,1),requires_grad=True)
    b=torch.zeros(1,requires_grad=True)
    return [w,b]
def l2_penalty(w):
    return 0.5*torch.sum(w.pow(2))

def linear_reg(X,w,b):
    return torch.matmul(X,w)+b
def mse_loss(y_hat,y):
    return (y_hat-y)**2/2
def sgd(params,lr,batch_size):
    for params in params:
        params.data-=lr*params.grad/batch_size
        params.grad.zero_()
def evaluate_loss(net, data_iter, loss):
    total_loss, total_samples = 0.0, 0
    for X, y in data_iter:
        l = loss(net(X), y)
        total_loss += l.sum().item()
        total_samples += y.numel()
    return total_loss / total_samples
n_train,n_test,num_inputs,batch_size=20,100,200,5
true_w,true_b=torch.ones((num_inputs,1))*0.01,0.05
train_data=synthetic_data(true_w,true_b,n_train)
test_data=synthetic_data(true_w,true_b,n_test)
train_iter=load_array(train_data,batch_size)
test_iter=load_array(test_data,batch_size,is_train=False)
w,b=init_params(num_inputs)
net=lambda X:linear_reg(X,w,b)
loss=mse_loss
num_epochs,lr,lambd=10,0.05,3
#animator=SimpleAnimator()
for epoch in range(num_epochs):
    for X,y in train_iter:
        l=loss(net(X),y)+lambd*l2_penalty(w)
        l.sum().backward()
        sgd([w,b],lr,batch_size)
    if (epoch+1)%5==0:
        train_loss=evaluate_loss(net,train_iter,loss)
        test_loss=evaluate_loss(net,test_iter,loss)
        #animator.add(epoch+1,train_loss,test_loss)
        print(f"Epoch {epoch+1}: Train Loss: {train_loss:.4f},test Loss: {test_loss:.4f}")
print('w的L2范数是:', torch.norm(w).item())
plt.show()

4.2 简单实现权重衰减

import torch
from torch import nn
from torch.utils.data import TensorDataset,DataLoader
import matplotlib.pyplot as plt
def synthetic_data(w,b,num_inputs):
    X=torch.normal(0,1,size=(num_inputs,w.shape[0]))
    y=X@w+b
    y+=torch.normal(0,0.1,size=y.shape)
    return X,y
def load_array(data,batch_size,is_train=True):
    dataset=TensorDataset(*data)
    return DataLoader(dataset,batch_size=batch_size,shuffle=is_train)
def init_params(num_inputs):
    w=torch.normal(0,1,size=(num_inputs,1),requires_grad=True)
    b=torch.zeros(1,requires_grad=True)
    return [w,b]
def l2_penalty(w):
    return 0.5*torch.sum(w.pow(2))
def linear_reg(X,w,b):
    return torch.matmul(X,w)+b
def mse_loss(y_hat,y):
    return ((y_hat-y)**2).sum()/2
def evaluate_loss(net, data_iter, loss):
    total_loss, total_samples = 0.0, 0
    for X, y in data_iter:
        l = loss(net(X), y)
        total_loss += l.item()*y.shape[0]
        total_samples += y.numel()
    return total_loss / total_samples
n_train,n_test,num_inputs,batch_size=20,100,200,5
true_w,true_b=torch.ones((num_inputs,1))*0.01,0.05
train_data=synthetic_data(true_w,true_b,n_train)
test_data=synthetic_data(true_w,true_b,n_test)
train_iter=load_array(train_data,batch_size)
test_iter=load_array(test_data,batch_size,is_train=False)
w,b=init_params(num_inputs)
net=lambda X:linear_reg(X,w,b)
loss=mse_loss
num_epochs,lr,lambd=100,0.001,3
optimizer=torch.optim.SGD([w,b],lr=lr,weight_decay=0.001)
#animator=SimpleAnimator()
for epoch in range(num_epochs):
    for X,y in train_iter:
        optimizer.zero_grad()
        l=loss(net(X),y)
        l.backward()
        #sgd([w,b],lr,batch_size)
        optimizer.step() 
    if (epoch+1)%5==0:
        train_loss=evaluate_loss(net,train_iter,loss)
        test_loss=evaluate_loss(net,test_iter,loss)
        #animator.add(epoch+1,train_loss,test_loss)
        print(f"Epoch {epoch+1}: Train Loss: {train_loss:.4f},test Loss: {test_loss:.4f}")
print('w的L2范数是:', torch.norm(w).item())
plt.show()