接上一篇:
LayerNorm的图是不是画错了
这里手动计算 LN,本篇把我前两天闲的没事干写的验证代码放上了,还是上一篇的问题,有木有大佬解决一下我上一篇的问题,LayerNorm的图是画错了,还是我理解错了
1. 手动计算BN
python
import torch
import torch.nn as nn
torch.manual_seed(1107)
# x = torch.arange(4).float().reshape(1, 2, 1, 2) # 假设的输入
x = torch.rand(3, 32, 30, 32)
# 实例化BatchNorm2d
m = nn.BatchNorm2d(32, momentum=1)
m.train() # 设置为评估模式
# m.running_var = 0
# 使用BatchNorm2d计算
y = m(x)
# 手动计算BatchNorm2d
x_mean = x.mean(dim=[0, 2, 3], keepdim=True)
x_var = x.var(dim=[0, 2, 3], keepdim=True, unbiased=False)
eps = m.eps
y_manual = (x - x_mean) / ((x_var + eps).sqrt())
# 检查两种方法的输出是否一致
# print("使用BatchNorm2d的结果:", y)
# print("手动计算的结果:", y_manual)
print("结果是否一致:", torch.allclose(y, y_manual, atol=1e-6))
2. 手动计算GN
python
import torch
import torch.nn as nn
torch.manual_seed(1107)
num_channels = 64 # 确保这个数可以被num_groups整除
# 假设x的形状是(B, C, H, W),这里我们按照通常的卷积神经网络输入布局
x = torch.rand(32, num_channels, 256, 256) # 修改x的形状以适配GroupNorm的输入需求
# 定义组数G,每组的通道数C/G需要是整数
num_groups = 32
m = nn.GroupNorm(
num_groups=num_groups, num_channels=num_channels, eps=1e-5, affine=False
)
m.eval() # 设置为评估模式
y = m(x)
# 手动计算GroupNorm
C_per_group = num_channels // num_groups
x = x.view(32, num_groups, C_per_group, 256, 256) # 重塑x以便可以对每组进行操作
x_mean = x.mean(dim=[2, 3, 4], keepdim=True)
x_var = x.var(dim=[2, 3, 4], keepdim=True, unbiased=False)
eps = m.eps
y_manual = (x - x_mean) / ((x_var + eps).sqrt())
y_manual = y_manual.view(32, num_channels, 256, 256) # 将y_manual的形状重塑回原始形状
print("结果是否一致:", torch.allclose(y, y_manual, atol=1e-6))
3. 手动计算IN
python
import torch
import torch.nn as nn
torch.manual_seed(1107)
# 假设x的形状是(B, C, H, W)
x = torch.rand(32, 256, 40, 40) # 添加一个维度以匹配四维输入
m = nn.InstanceNorm2d(256, affine=False, momentum=1)
m.eval() # 设置为评估模式
y = m(x)
# 手动计算LayerNorm
x_mean = x.mean(dim=[2, 3], keepdim=True)
x_var = x.var(dim=[2, 3], keepdim=True, unbiased=False)
eps = m.eps
y_manual = (x - x_mean) / ((x_var + eps).sqrt())
print("结果是否一致:", torch.allclose(y, y_manual, atol=1e-6))