深度学习32Transformer、BERT、

1. Transformer

2. Transformer

import math
import os
import pandas as pd
import torch
from torch import nn
from d2l import torch as d2l

# 基于位置的前馈网络
class PositionWiseFFN(nn.Module):
    def __init__(self, ffn_num_input, ffn_num_hiddens, ffn_num_outputs, **kwargs):
        super(PositionWiseFFN, self).__init__(**kwargs)
        # 第一个全连接层，将输入维度从ffn_num_input映射到ffn_num_hiddens
        self.dense1 = nn.Linear(ffn_num_input, ffn_num_hiddens)
        # ReLU激活函数，增加非线性特性
        self.relu = nn.ReLU()
        # 第二个全连接层，将输入维度从ffn_num_hiddens映射到ffn_num_outputs
        self.dense2 = nn.Linear(ffn_num_hiddens, ffn_num_outputs)

    def forward(self, X):
        # 前向传播过程
        # 通过第一个全连接层，将输入X映射到隐藏层
        # 应用ReLU激活函数，增加非线性特性
        # 通过第二个全连接层，将隐藏层映射到输出层
        return self.dense2(self.relu(self.dense1(X)))

# 改变张量的最里层维度的尺寸
# 创建一个PositionWiseFFN实例，输入维度为4，隐藏层维度为4，输出维度为8
ffn = PositionWiseFFN(4, 4, 8)
# 设置模型为评估模式，不进行训练
ffn.eval()
# 创建一个形状为(2, 3, 4)的张量，所有元素都设置为1
# 输入张量通过前馈网络进行前向传播，得到输出张量
# 取第一个样本的输出张量
ffn(torch.ones((2,3,4)))[0]

tensor([[ 0.5166,  0.7577,  0.2357, -0.4697, -0.0154, -0.5645,  0.0913, -0.0949],
        [ 0.5166,  0.7577,  0.2357, -0.4697, -0.0154, -0.5645,  0.0913, -0.0949],
        [ 0.5166,  0.7577,  0.2357, -0.4697, -0.0154, -0.5645,  0.0913, -0.0949]],
       grad_fn=<SelectBackward0>)

# 对比不同维度的层归一化和批量归一化的效果
# 创建一个对最后一个维度进行层归一化的实例
ln = nn.LayerNorm(2)
# 创建一个对最后一个维度进行批量归一化的实例
bn = nn.BatchNorm1d(2)
# 创建一个形状为(2, 2)的张量
X = torch.tensor([[1, 2], [2, 3]], dtype=torch.float32)
# 对张量进行层归一化
# 对张量进行批量归一化
# 打印层归一化和批量归一化的结果
print('layer norm:', ln(X), '\nbatch norm:', bn(X))

layer norm: tensor([[-1.0000,  1.0000],
        [-1.0000,  1.0000]], grad_fn=<NativeLayerNormBackward0>) 
batch norm: tensor([[-1.0000, -1.0000],
        [ 1.0000,  1.0000]], grad_fn=<NativeBatchNormBackward0>)

# 使用残差连接和层归一化
class AddNorm(nn.Module):
    def __init__(self, normalized_shape, dropout, **kwargs):
        super(AddNorm, self).__init__(**kwargs)
        # 定义一个dropout层，用于随机丢弃部分神经元
        self.dropout = nn.Dropout(dropout)
        # 定义一个层归一化层，对输入进行归一化
        self.ln = nn.LayerNorm(normalized_shape)
        
    def forward(self, X, Y):
        # 前向传播过程
        # 使用残差连接：将dropout(Y)与X相加
        # 应用层归一化：对残差进行归一化
        return self.ln(self.dropout(Y) + X)

# 加法操作后输出张量的形状相同
# 创建一个AddNorm实例，输入的归一化维度为[3, 4]，dropout率为0.5
add_norm = AddNorm([3,4],0.5)
# 设置模型为评估模式，不进行训练
add_norm.eval()
# 创建一个形状为(2, 3, 4)的张量作为X
# 创建一个形状为(2, 3, 4)的张量作为Y
# 输入X和Y通过AddNorm进行前向传播
# 打印输出结果的形状
add_norm(torch.ones((2,3,4)), torch.ones((2,3,4))).shape

torch.Size([2, 3, 4])

# 实现编码器中的一个层
class EncoderBlock(nn.Module):
    def __init__(self, key_size, query_size, value_size, num_hiddens,
                norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,
                dropout, use_bias=False, **kwargs):
        super(EncoderBlock, self).__init__(**kwargs)
        # 多头注意力机制
        self.attention = d2l.MultiHeadAttention(key_size, query_size,
                                               value_size, num_hiddens,
                                               num_heads, dropout, use_bias)   
        # 第一个残差连接和层归一化模块
        self.addnorm1 = AddNorm(norm_shape, dropout)
        # 位置前馈网络
        self.ffn = PositionWiseFFN(ffn_num_input, ffn_num_hiddens, num_hiddens)
        # 第二个残差连接和层归一化模块
        self.addnorm2 = AddNorm(norm_shape, dropout)
        
    def forward(self, X, valid_lens):
        # 前向传播过程
        # 多头注意力机制：计算注意力权重并应用于X
        # 第一个残差连接和层归一化模块：将注意力输出与X相加并进行归一化
        Y = self.addnorm1(X, self.attention(X, X, X, valid_lens))
        # 位置前馈网络：对第一个残差连接和层归一化模块的输出进行位置前馈网络操作
        # 第二个残差连接和层归一化模块：将位置前馈网络输出与第一个残差连接和层归一化模块的输出相加并进行归一化
        return self.addnorm2(Y, self.ffn(Y))

# Transformer编码器中的任何层都不会改变其输入的状态
# 创建一个形状为(2, 100, 24)的张量作为输入X
X = torch.ones((2,100,24))
# 创建一个形状为(2,)的张量作为有效长度
valid_lens = torch.tensor([3,2])
# 创建一个EncoderBlock实例
encoder_blk = EncoderBlock(24,24,24,24,[100,24],24,48,8,0.5)
# 设置模型为评估模式，不进行训练
encoder_blk.eval()
# 输入X和有效长度通过EncoderBlock进行前向传播
# 打印输出结果的形状
encoder_blk(X, valid_lens).shape

torch.Size([2, 100, 24])

# Transformer编码器
class TransformerEncoder(d2l.Encoder):
    def __init__(self, vocab_size, key_size, query_size, value_size,
                num_hiddens, norm_shape, ffn_num_input, ffn_num_hiddens,
                num_heads, num_layers, dropout, use_bias=False, **kwargs): 
        super(TransformerEncoder, self).__init__(**kwargs)
        # 词嵌入层
        self.num_hiddens = num_hiddens
        # 位置编码层
        self.embedding = nn.Embedding(vocab_size, num_hiddens)
        # 创建多个编码器块组成的序列
        self.pos_encoding = d2l.PositionalEncoding(num_hiddens, dropout)
        # 创建了一个空的顺序容器self.blks，用于存储多个编码器块
        self.blks = nn.Sequential()
        # 通过循环迭代的方式，逐个添加编码器块到顺序容器self.blks中
        for i in range(num_layers):
            # 使用self.blks.add_module()方法将一个新的编码器块添加到顺序容器中
            self.blks.add_module(
                "block" + str(i),
                EncoderBlock(key_size, query_size, value_size, num_hiddens,
                             norm_shape, ffn_num_input, ffn_num_hiddens,
                             num_heads, dropout, use_bias))
            
    def forward(self, X, valid_lens, *args):
        # 前向传播过程
        # 词嵌入层：对输入进行词嵌入操作
        X = self.pos_encoding(self.embedding(X) * math.sqrt(self.num_hiddens))   
        # 用于存储每个编码器块中的注意力权重
        self.attention_weights = [None] * len(self.blks)
        for i, blk in enumerate(self.blks):
            # 编码器块：将词嵌入结果传入编码器块进行处理
            X = blk(X, valid_lens)
            # 将每个编码器块的注意力权重存储到self.attention_weights列表中的对应位置
            self.attention_weights[i] = blk.attention.attention.attention_weights
        return X

# 创建一个两层的Transformer编码器
encoder = TransformerEncoder(200, 24, 24, 24, 24, [100, 24], 24, 48, 8, 2, 0.5)     
# 将编码器设置为评估模式
encoder.eval()
# 对输入数据进行前向传播，获取输出的形状
encoder(torch.ones((2,100),dtype=torch.long),valid_lens).shape

torch.Size([2, 100, 24])

# Transformer解码器也是由多个相同的层组成
class DecoderBlock(nn.Module):
    """解码器中第 i 个块"""
    def __init__(self, key_size, query_size, value_size, num_hiddens,
                norm_shape, ffn_num_input, ffn_num_hiddens, num_heads,
                dropout, i, **kwargs):
        super(DecoderBlock, self).__init__(**kwargs)
        self.i = i
        # 第一个多头注意力机制
        self.attention1 = d2l.MultiHeadAttention(key_size, query_size,
                                                value_size, num_hiddens,
                                                num_heads, dropout)
        # 第一个残差连接和层归一化模块
        self.addnorm1 = AddNorm(norm_shape, dropout)
        # 第二个多头注意力机制
        self.attention2 = d2l.MultiHeadAttention(key_size, query_size,
                                                value_size, num_hiddens,
                                                num_heads, dropout)
        # 第二个残差连接和层归一化模块
        self.addnorm2 = AddNorm(norm_shape, dropout)
        # 位置前馈网络
        self.ffn = PositionWiseFFN(ffn_num_input, ffn_num_hiddens, num_hiddens)   
        # 第三个残差连接和层归一化模块
        self.addnorm3 = AddNorm(norm_shape, dropout)
        
    def forward(self, X, state):
        # 从state中提取编码器输出的相关信息
        enc_outputs, enc_valid_lens = state[0], state[1]
        # 检查状态中当前解码器块的键-值对是否为None
        if state[2][self.i] is None:
            key_values = X
        else:
            # 将当前解码器块的键-值对与输入X在维度1上进行拼接
            key_values = torch.cat((state[2][self.i], X), axis=1)
        # 更新状态中当前解码器块的键-值对
        state[2][self.i] = key_values
        if self.training:
            batch_size, num_steps, _ = X.shape
            # 生成一个从1到num_steps的张量，用于表示解码器输入的有效长度
            dec_valid_lens = torch.arange(1, num_steps + 1,
                                         device = X.device).repeat(batch_size, 1)   
        else:
            # 在评估模式下，解码器输入的有效长度为None
            dec_valid_lens = None
        # 第一个多头注意力机制的输出
        X2 = self.attention1(X, key_values, key_values, dec_valid_lens)
        # 第一个残差连接和层归一化模块的输出
        Y = self.addnorm1(X, X2)
        # 第二个多头注意力机制的输出
        Y2 = self.attention2(Y, enc_outputs, enc_outputs, enc_valid_lens)
        # 第二个残差连接和层归一化模块的输出
        Z = self.addnorm2(Y, Y2)
        # 最终输出经过第三个残差连接和层归一化模块处理，返回结果和更新后的状态
        return self.addnorm3(Z, self.ffn(Z)), state

# 编码器和解码器的特征维度都是num_hiddens
# 创建一个解码器块实例，特征维度为24
decoder_blk = DecoderBlock(24, 24, 24, 24, [100, 24], 24, 48, 8, 0.5, 0)
# 将解码器块设置为评估模式
decoder_blk.eval()
# 创建一个形状为(2, 100, 24)的输入张量X
X = torch.ones((2, 100, 24))
# 使用编码器块对输入X进行编码，得到编码器的输出和有效长度
state = [encoder_blk(X, valid_lens), valid_lens, [None]]
# 输入X和状态state通过解码器块进行前向传播，得到输出张量
# 打印输出张量的形状
decoder_blk(X, state)[0].shape

torch.Size([2, 100, 24])

# Transform解码器
class TransformerDecoder(d2l.AttentionDecoder):
    def __init__(self, vocab_size, key_size, query_size, value_size,
                num_hiddens, norm_shape, ffn_num_input, ffn_num_hiddens,
                num_heads, num_layers, dropout, **kwargs):
        super(TransformerDecoder, self).__init__(**kwargs)
        # 初始化参数
        self.num_hiddens = num_hiddens
        self.num_layers = num_layers
        # 词嵌入层
        self.embedding = nn.Embedding(vocab_size, num_hiddens)
        # 位置编码层
        self.pos_encoding = d2l.PositionalEncoding(num_hiddens, dropout)
        # 创建多个解码器块组成的序列
        self.blks = nn.Sequential()
        # 通过循环迭代的方式，逐个添加解码器块到顺序容器self.blks中
        for i in range(num_layers):
            # 使用self.blks.add_module()方法将一个新的解码器块添加到顺序容器中
            self.blks.add_module(
                "block" + str(i),
                DecoderBlock(key_size, query_size, value_size, num_hiddens,
                             norm_shape, ffn_num_input, ffn_num_hiddens,
                             num_heads, dropout, i))
        # 全连接层，将解码器块的输出转换为词汇表大小的输出
        self.dense = nn.Linear(num_hiddens, vocab_size)
        
    def init_state(self, enc_outputs, enc_valid_lens, *args):
        # 初始化解码器状态
        return [enc_outputs, enc_valid_lens, [None] * self.num_layers]
    
    def forward(self, X, state):
        # 前向传播过程
        # 词嵌入层：对输入进行词嵌入操作
        X = self.pos_encoding(self.embedding(X) * math.sqrt(self.num_hiddens))  
        # 用于存储每个解码器块中的注意力权重
        self._attention_weights = [[None] * len(self.blks) for _ in range(2)] 
        for i, blk in enumerate(self.blks):
            # 解码器块：将词嵌入结果传入解码器块进行处理
            X, state = blk(X, state)
            # 将每个解码器块的注意力权重存储到self._attention_weights列表中的对应位置
            self._attention_weights[0][i] = blk.attention1.attention.attention_weights   
            self._attention_weights[1][i] = blk.attention2.attention.attention_weights
        # 全连接层：将解码器块的输出转换为词汇表大小的输出
        return self.dense(X), state
    
    @property
    def attention_weights(self):
        # 返回注意力权重
        return self._attention_weights

def read_data_nmt():
    """载入 "英语-法语" 数据集 """
    # 下载并解压数据集
    data_dir = d2l.download_extract('fra-eng')
    # 打开文件并读取数据
    with open(os.path.join(data_dir, 'fra.txt'), 'r', encoding='utf-8') as f:
        return f.read()

def preprocess_nmt(text):
    """预处理 "英语-法语" 数据集"""
    def no_space(char, prev_char):
        return char in set(',.!?') and prev_char != ''
    # 替换特殊字符并转换为小写
    text = text.replace('\u202f', ' ').replace('\xa0',' ').lower()
    # 在标点符号前添加空格，以便于分词
    out = [
        ' ' + char if i > 0 and no_space(char, text[i - 1]) else char
        for i, char in enumerate(text)]
    # 将字符列表拼接为字符串
    return ''.join(out)

def tokenize_nmt(text, num_examples=None):
    """词元化 "英语-法语" 数据数据集 """
    source, target = [], []
    # 按行遍历文本
    for i, line in enumerate(text.split('\n')):
        # 检查是否达到指定的样本数量
        if num_examples and i > num_examples:
            break
        # 按制表符分割行，将源语言和目标语言分别存储到source和target列表中
        parts = line.split('\t')
        if len(parts) == 2:
            source.append(parts[0].split(' '))
            target.append(parts[1].split(' '))
    return source, target

def truncate_pad(line, num_steps, padding_token):
    """截断或填充文本序列"""
    # 检查文本序列长度是否超过指定的最大长度
    if len(line) > num_steps:
        # 如果超过最大长度，则截断序列，只保留前面的num_steps个元素
        return line[:num_steps]
    # 如果未超过最大长度，则使用padding_token进行填充，使序列长度达到num_steps
    return line + [padding_token] * (num_steps - len(line))

def build_array_nmt(lines, vocab, num_steps):
    """将机器翻译的文本序列转换成小批量"""
    lines = [vocab[l] for l in lines]
    lines = [l + [vocab['<eos>']] for l in lines]
    array = torch.tensor([ truncate_pad(l, num_steps, vocab['<pad>']) for l in lines ])
    valid_len = (array != vocab['<pad>']).type(torch.int32).sum(1)
    return array, valid_len

def load_data_nmt(batch_size, num_steps, num_examples=600):
    """返回翻译数据集的迭代器和词汇表"""
    # 载入并预处理文本数据
    text = preprocess_nmt(read_data_nmt())
    # 对文本数据进行词元化
    source, target = tokenize_nmt(text, num_examples)
    # 构建源语言和目标语言的词汇表
    src_vocab = d2l.Vocab(source, min_freq=2,
                         reserved_tokens=['<pad>','<bos>','<eos>'])
    tgt_vocab = d2l.Vocab(target, min_freq=2,
                         reserved_tokens=['<pad>','<bos>','<eos>'])
    # 将词元化后的文本数据转换为数值数组，并计算有效长度
    src_array, src_valid_len = build_array_nmt(source, src_vocab, num_steps)  
    tgt_array, tgt_valid_len = build_array_nmt(target, tgt_vocab, num_steps)
    # 构建数据集迭代器
    data_arrays = (src_array, src_valid_len, tgt_array, tgt_valid_len)
    data_iter = d2l.load_array(data_arrays, batch_size)
    # 返回数据集迭代器和词汇表
    return data_iter, src_vocab, tgt_vocab

# 训练
# 设置隐藏单元数、层数、丢弃率、批量大小、序列长度
num_hiddens, num_layers, dropout, batch_size, num_steps = 32, 2, 0.1, 64, 10
# 设置学习率、训练轮数和设备
lr, num_epochs, device = 0.005, 200, d2l.try_gpu()
# 设置前馈神经网络的输入维度、隐藏单元数和注意力头数
ffn_num_input, ffn_num_hiddens, num_heads = 32, 64, 4
# 设置键、查询和值的维度
key_size, query_size, value_size = 32, 32, 32
# 设置规范化层的形状
norm_shape = [32]
# 加载训练数据集和词汇表
train_iter, src_vocab, tgt_vocab = load_data_nmt(batch_size, num_steps)   
# 创建Transformer编码器和解码器
encoder = TransformerEncoder(len(src_vocab), key_size, query_size, value_size,
                            num_hiddens, norm_shape, ffn_num_input,
                            ffn_num_hiddens, num_heads, num_layers, dropout)
decoder = TransformerDecoder(len(tgt_vocab), key_size, query_size, value_size,
                            num_hiddens, norm_shape, ffn_num_input,
                            ffn_num_hiddens, num_heads, num_layers, dropout)
# 创建Encoder-Decoder模型
net = d2l.EncoderDecoder(encoder, decoder)
# 使用序列到序列模型进行训练
d2l.train_seq2seq(net, train_iter, lr, num_epochs, tgt_vocab, device)  

loss 0.029, 4632.7 tokens/sec on cuda:0

# 将一些英语句子翻译成法语
# 定义一些英语句子和对应的法语翻译
engs = ['go .', "i lost .", 'he\'s calm .', 'i\'m home .']
fras = ['va !', 'j\'ai perdu .', 'il est calme .', 'je suis chez moi .']   
# 遍历每个英语句子和对应的法语翻译
for eng, fra in zip(engs, fras):
    # 使用训练好的模型进行翻译，并获取解码器注意力权重序列
    translation, dec_attention_weight_seq = d2l.predict_seq2seq(
        net, eng, src_vocab, tgt_vocab, num_steps, device, True)
    # 打印翻译结果和BLEU评分
    print(f'{eng} => {translation}, ',
         f'bleu {d2l.bleu(translation, fra, k=2):.3f}')

go . => va  !,  bleu 0.000
i lost . => j’ai <unk> .,  bleu 0.000
he's calm . => il est calme .,  bleu 1.000
i'm home . => je suis chez moi .,  bleu 1.000

# 可视化Transformer的注意力权重
# 将编码器的注意力权重拼接起来，并调整形状
enc_attention_weights = torch.cat(net.encoder.attention_weights,0).reshape((
    num_layers, num_heads, -1, num_steps))   
# 打印注意力权重的形状
enc_attention_weights.shape

torch.Size([2, 4, 10, 10])

# 使用d2l.show_heatmaps函数展示注意力权重的热图
d2l.show_heatmaps(enc_attention_weights.cpu(), xlabel='Key positions',
                 ylabel = 'Query position',
                 titles = ['Head %d' % i
                          for i in range(1, 5)], figsize=(7, 3.5))  

b'\r\n\r\n\r\n

# 为了可视化解码器的自注意力权重和 "编码器-解码器" 的注意力权重，我们需要完成更多的数据操作工作
# 将解码器注意力权重列表转换为二维列表。通过迭代解码器注意力权重序列的步骤、注意力头、块和头部，将注意力权重提取为二维列表      
dec_attention_weights_2d = [
    head[0].tolist() for step in dec_attention_weight_seq for attn in step
    for blk in attn for head in blk]
# 将注意力权重的二维列表转换为填充了缺失值的张量
dec_attention_weights_filled = torch.tensor(
    pd.DataFrame(dec_attention_weights_2d).fillna(0.0).values)   
# 重新调整张量的形状
dec_attention_weights = dec_attention_weights_filled.reshape(
    (-1, 2, num_layers, num_heads, num_steps))
# 提取解码器的自注意力权重和 "编码器-解码器" 的注意力权重
dec_self_attention_weights, dec_inter_attention_weights = dec_attention_weights.permute(1,2,3,0,4)   
# 输出解码器的自注意力权重和 "编码器-解码器" 的注意力权重的形状
dec_self_attention_weights.shape, dec_inter_attention_weights.shape

(torch.Size([2, 4, 6, 10]), torch.Size([2, 4, 6, 10]))

# 可视化解码器的自注意力权重
d2l.show_heatmaps(dec_self_attention_weights[:, :, :, :len(translation.split()) + 1], 
             xlabel = 'Key positions', ylabel = 'Query position',
             titles = ['Head %d' % i for i in range(1,5)], figsize=(7,3.5))

b'\r\n\r\n\r\n

# 输出序列的查询不会与输入序列中填充位置的标记进行注意力计算
# 注意力权重表示输出序列的查询在编码器输出序列的键位置上的注意力分布
d2l.show_heatmaps(dec_inter_attention_weights, xlabel = 'Key positions',
             ylabel = 'Query positions',
             titles = ['Head %d' % i
                      for i in range(1, 5)], figsize=(7, 3.5))   

b'\r\n\r\n\r\n

1. BERT

2. BERT

import torch
from torch import nn
from d2l import torch as d2l

# Input Representation
def get_tokens_and_segments(tokens_a, tokens_b=None):
    """Get tokens of the BERT input sequence and their segment IDs"""   
    # 添加特殊标记，并连接第一个句子的标记
    tokens = ['<cls>'] + tokens_a + ['<seq>']
    # 第一个句子的段ID都为0
    segments = [0] * (len(tokens_a) + 2)
    if tokens_b is not None:
        # 如果存在第二个句子，则连接第二个句子的标记
        tokens += tokens_b + ['<seq>']
        # 第二个句子的段ID都为1
        segments += [1] * (len(tokens_b) + 1)
    # 返回转换后的标记列表和段ID列表
    return tokens, segments

# BERTEncoder class
class BERTEncoder(nn.Module):
    """BERT encoder."""
    def __init__(self, vocab_size, num_hiddens, norm_shape, ffn_num_input,
                ffn_num_hiddens, num_heads, num_layers, dropout,
                max_len=1000, key_size=768, query_size=768, value_size=768,
                **kwargs):
        super(BERTEncoder, self).__init__(**kwargs)
        # 标记嵌入层
        self.token_embedding = nn.Embedding(vocab_size, num_hiddens)
        # 段嵌入层
        self.segment_embedding = nn.Embedding(2, num_hiddens)
        # BERT编码器块的序列容器
        self.blks = nn.Sequential()
        for i in range(num_layers):
            # 添加BERT编码器块
            self.blks.add_module(f"{i}", d2l.EncoderBlock(
                key_size, query_size, value_size, num_hiddens, norm_shape,
                ffn_num_input, ffn_num_hiddens, num_heads, dropout, True))
        # 位置嵌入参数
        self.pos_embedding = nn.Parameter(torch.randn(1, max_len, num_hiddens))   
    
    def forward(self, tokens, segments, valid_lens):
        # 计算输入序列的嵌入表示
        X = self.token_embedding(tokens) + self.segment_embedding(segments)  
        # 添加位置嵌入
        X = X + self.pos_embedding.data[:, :X.shape[1], :]
        for blk in self.blks:
            # 通过BERT编码器块进行编码
            X = blk(X, valid_lens)
        return X

class BERTEncoder(nn.Module):
    """BERT encoder."""
    def __init__(self, vocab_size, num_hiddens, norm_shape, ffn_num_input,
                 ffn_num_hiddens, num_heads, num_layers, dropout,
                 max_len=1000, key_size=768, query_size=768, value_size=768,
                 **kwargs):
        super(BERTEncoder, self).__init__(**kwargs)
        # 标记嵌入层
        self.token_embedding = nn.Embedding(vocab_size, num_hiddens)
        # 段嵌入层
        self.segment_embedding = nn.Embedding(2, num_hiddens)
        # BERT编码器块的序列容器
        self.blks = nn.Sequential()
        for i in range(num_layers):
            # 添加BERT编码器块
            self.blks.add_module(f"{i}", d2l.EncoderBlock(
                key_size, query_size, value_size, num_hiddens, norm_shape,
                ffn_num_input, ffn_num_hiddens, num_heads, dropout, True))
        # 位置嵌入参数
        self.pos_embedding = nn.Parameter(torch.randn(1, max_len,
                                                      num_hiddens))

    def forward(self, tokens, segments, valid_lens):
        # 计算嵌入表示
        X = self.token_embedding(tokens) + self.segment_embedding(segments)
        # 添加位置嵌入
        X = X + self.pos_embedding.data[:, :X.shape[1], :]
        for blk in self.blks:
            # 进行编码
            X = blk(X, valid_lens)
        return X

# Inference of BERTEncoder
# 定义BERT编码器的参数
# vocab_size: 词汇表大小
# num_hiddens: 隐藏单元数
# ffn_num_hiddens: 前馈神经网络隐藏层大小
# num_heads: 注意力头数
# norm_shape: 规范化层的形状
# ffn_num_input: 前馈神经网络输入大小
# num_layers: 编码器层数
# dropout: dropout概率
vocab_size, num_hiddens, ffn_num_hiddens, num_heads = 10000, 768, 1024, 4
norm_shape, ffn_num_input, num_layers, dropout = [768], 768, 2, 0.2
# 创建BERT编码器实例
encoder = BERTEncoder(vocab_size, num_hiddens, norm_shape, ffn_num_input,
                     ffn_num_hiddens, num_heads, num_layers, dropout)
# 随机生成标记
tokens = torch.randint(0, vocab_size, (2,8))
# 创建段向量
segments = torch.tensor([[0, 0, 0, 0, 1, 1, 1, 1], [0, 0, 0, 1, 1, 1, 1, 1]])  
# 进行BERT编码
encoded_X = encoder(tokens, segments, None)
# 输出编码后的表示结果的形状
encoded_X.shape

torch.Size([2, 8, 768])

# Masked Language Modeling
class MaskLM(nn.Module):
    """The masked language model task of BERT."""
    def __init__(self, vocab_size, num_hiddens, num_inputs=768, **kwargs):
        super(MaskLM, self).__init__(**kwargs)
        self.mlp = nn.Sequential(nn.Linear(num_inputs, num_hiddens),  # 全连接层：输入维度为num_inputs，输出维度为num_hiddens
                                nn.ReLU(), # ReLU激活函数
                                nn.LayerNorm(num_hiddens),  # LayerNorm层，归一化输入
                                nn.Linear(num_hiddens, vocab_size)) # 全连接层：输入维度为num_hiddens，输出维度为vocab_size
    
    def forward(self, X, pred_positions):
        # 预测位置数量
        num_pred_positions = pred_positions.shape[1]
        # 重塑预测位置张量为一维向量
        pred_positions = pred_positions.reshape(-1)
        # 批量大小
        batch_size = X.shape[0]
        # 创建批量索引向量
        batch_idx = torch.arange(0, batch_size)
        # 重复批量索引以匹配预测位置索引
        batch_idx = torch.repeat_interleave(batch_idx, num_pred_positions)  
        # 从X中提取被掩盖的输入
        masked_X = X[batch_idx, pred_positions]
        # 重塑masked_X张量的形状
        masked_X = masked_X.reshape((batch_size, num_pred_positions, -1))
        # 将masked_X传递给MLP网络，得到MLM任务的预测结果
        mlm_Y_hat = self.mlp(masked_X)
        return mlm_Y_hat

# The forward inference of MaskLM
# 创建MaskLM模型实例
mlm = MaskLM(vocab_size, num_hiddens)
# 创建MLM任务的预测位置张量
mlm_positions = torch.tensor([[1,5,2],[6,1,5]])
# 输入编码后的文本和预测位置，得到MLM任务的预测结果
mlm_Y_hat = mlm(encoded_X, mlm_positions)
# 打印MLM任务的预测结果的形状
mlm_Y_hat.shape

torch.Size([2, 3, 10000])

# 创建MLM任务的目标张量
mlm_Y = torch.tensor([[7,8,9],[6,1,5]])
# 创建交叉熵损失函数实例
loss = nn.CrossEntropyLoss(reduction='none')
# 计算MLM任务的损失
mlm_l = loss(mlm_Y_hat.reshape((-1, vocab_size)), mlm_Y.reshape(-1))
# 打印MLM任务的损失的形状
mlm_l.shape

torch.Size([6])

# Next Sentence Prediction
class NextSentencePred(nn.Module):
    """The next sentence prediction task of BERT."""
    def __init__(self, num_inputs, **kwargs):
        super(NextSentencePred, self).__init__(**kwargs)
        # 全连接层用于预测下一句的概率
        self.output = nn.Linear(num_inputs, 2)
    
    def forward(self, X):
        # 输出下一句的预测结果
        return self.output(X)
# The forward inference of an NextSentencePred
# 将encoded_X展平为二维张量
encoded_X = torch.flatten(encoded_X, start_dim=1)
# 创建NextSentencePred实例，输入大小为encoded_X的最后一维大小
nsp = NextSentencePred(encoded_X.shape[-1])
# 使用nsp对encoded_X进行前向传播，得到下一句预测的结果
nsp_Y_hat = nsp(encoded_X)
# 打印下一句预测结果的形状
nsp_Y_hat.shape

torch.Size([2, 2])

# 创建下一句预测的标签张量
nsp_y = torch.tensor([0,1])
# 计算下一句预测的损失
nsp_l = loss(nsp_Y_hat, nsp_y)
# 打印下一句预测损失的形状
nsp_l.shape

torch.Size([2])

# Putting All Things Together
class BERTModel(nn.Module):
    """The BERT model."""
    def __init__(self, vocab_size, num_hiddens, norm_shape, ffn_num_input,
                ffn_num_hiddens, num_heads, num_layers, dropout,
                max_len=1000, key_size=768, mlm_in_features=768,
                nsp_in_features=768):
        super(BERTModel, self).__init__()
        # BERT编码器
        self.encoder = BERTEncoder(vocab_size, num_hiddens, norm_shape,
                                  ffn_num,input, ffn_num_hiddens, num_heads, num_layers,
                                  dropout, max_len=max_len, key_size=key_size)
        # 隐藏层
        self.hidden = nn.Sequential(nn.Linear(hid_in_features, num_hiddens), nn.Tanh())   
        # 掩码语言模型
        self.mlm = MaskLM(vocab_size, num_hiddens, mlm_in_features)
        # 下一句预测模型
        self.nsp = NextSentencePred(nsp_in_features)
        
    def forward(self, tokens, segments, valid_lens=None, pred_positions=None):
        # 使用编码器对输入进行编码
        encoded_X = self.encoder(tokens, segments, valid_lens)
        if pred_positions is not None:
            # 如果传入了pred_positions参数，则调用掩码语言模型进行预测
            mlm_Y_hat = self.mlm(encoded_X, pred_positions)
        else:
            mlm_Y_hat = None
        # 将encoded_X的第一个位置的隐藏表示通过隐藏层进行转换
        # 使用下一句预测模型进行预测
        nsp_Y_hat = self.nsp(self.hidden(encoded_X[:, 0, :]))
        return encoded_X, mlm_Y_hat, nsp_Y_hat

3. BERT预训练数据集

import os
import random
import torch
from d2l import torch as d2l

# The WikiText-2 dataset
# 将WikiText-2数据集添加到d2l的数据集中心
d2l.DATA_HUB['wikitext-2'] = ('https://s3.amazonaws.com/research.metamind.io/wikitext/'
                             'wikitext-2-v1.zip','3c914d17d80b1459be871a5039ac23e752a53cbe')   

def _read_wiki(data_dir):
    # 读取WikiText-2数据集的训练集文件
    file_name = os.path.join(data_dir, 'wiki.train.tokens')
    with open(file_name, 'r', encoding='utf-8') as f:
        # 逐行读取文件内容
        lines = f.readlines()
    # 将每行内容按句号分割成段落，并转换为小写
    paragraphs = [line.strip().lower().split(' . ')
                 for line in lines if len(line.split(' . ')) >= 2]
    # 随机打乱段落的顺序
    random.shuffle(paragraphs)
    return paragraphs

# Generating the Next Sentence Prediction Task
def _get_next_sentence(sentence, next_sentence, paragraphs):
    # 随机决定两个句子是否是下一个句子关系
    if random.random() < 0.5:
        is_next = True
    else:
        # 从随机选择的段落中选择一个句子作为下一个句子
        next_sentence = random.choice(random.choice(paragraphs))
        is_next = False
    return sentence, next_sentence, is_next

def _get_nsp_data_from_paragraph(paragraph, paragraphs, vocab, max_len):
    nsp_data_from_paragraph = []
    for i in range(len(paragraph) - 1):
        # 获取当前句子和下一个句子以及它们之间的关系
        tokens_a, tokens_b, is_next = _get_next_sentence(paragraph[i], paragraph[i + 1], paragraphs)
        # 如果两个句子加上特殊标记的长度超过了最大长度，则跳过该句对
        if len(tokens_a) + len(tokens_b) + 3 > max_len:
            continue
        # 获取句子的token和segment表示
        tokens, segments = d2l.get_tokens_and_segments(tokens_a, tokens_b)
        nsp_data_from_paragraph.append((tokens, segments, is_next))
    return nsp_data_from_paragraph

# Generating the Masked Language Modeling Task
def _replace_mlm_tokens(tokens, candidate_pred_positions, num_mlm_preds, vocab):
    # 生成用于MLM任务的输入tokens，同时返回预测位置和标签
    mlm_input_tokens = [token for token in tokens]
    pred_positions_and_labels = []
    random.shuffle(candidate_pred_positions)
    for mlm_pred_position in candidate_pred_positions:
        # 如果已经预测了足够数量的位置，则结束
        if len(pred_positions_and_labels) >= num_mlm_preds:
            break
        masked_token = None
        # 随机决定当前位置是否进行mask操作
        if random.random() < 0.8:
            masked_token = '<mask>'
        else:
            # 随机决定是替换为当前token还是随机选择一个token作为替换
            if random.random() < 0.5:
                masked_token = tokens[mlm_pred_position]
            else:
                masked_token = random.randint(0, len(vocab) - 1)
        mlm_input_tokens[mlm_pred_position] = masked_token
        pred_positions_and_labels.append((mlm_pred_position, tokens[mlm_pred_position]))  
    return mlm_input_tokens, pred_positions_and_labels

def _get_mlm_data_from_tokens(tokens, vocab):
    candidate_pred_positions = []
    for i, token in enumerate(tokens):
        # 跳过特殊标记的位置
        if token in ['<cls>','<sep>']:
            continue
        candidate_pred_positions.append(i)
    num_mlm_preds = max(1, round(len(tokens) * 0.15))    
    mlm_input_tokens, pred_positions_and_labels = _replace_mlm_tokens(
        tokens, candidate_pred_positions, num_mlm_preds, vocab)
    # 按照预测位置的顺序进行排序
    pred_positions_and_labels = sorted(pred_positions_and_labels, key=lambda x:x[0])   
    pred_positions = [v[0] for v in pred_positions_and_labels]
    mlm_pred_labels = [v[1] for v in pred_positions_and_labels]
    return vocab[mlm_input_tokens], pred_positions, vocab[mlm_pred_labels]     

# Append the special "<mask>" tokens to the inputs
def _pad_bert_inputs(examples, max_len, vocab):
    # 计算最大的预测位置数量
    max_num_mlm_preds = round(max_len * 0.15)
    all_token_ids, all_segments, valid_lens, = [], [], []
    all_pred_positions, all_mlm_weights, all_mlm_labels = [], [], []
    nsp_labels = []
    # 遍历每个样本
    for (token_ids, pred_positions, mlm_pred_label_ids, segments, is_next) in examples:
        # 将输入 token 填充到指定长度，并转换为张量
        all_token_ids.append(torch.tensor(token_ids + [vocab['<pad>']] * (max_len - len(token_ids)), dtype=torch.long))     
        # 将输入 segment 填充到指定长度，并转换为张量
        all_segments.append(torch.tensor(segments + [0] * (max_len - len(segments)), dtype=torch.long))
        # 记录有效长度，即实际 token 的数量
        valid_lens.append(torch.tensor(len(token_ids), dtype=torch.float32))
        # 将预测位置填充到最大预测位置数量，并转换为张量
        all_pred_positions.append(torch.tensor(pred_positions + [0] * (
            max_num_mlm_preds - len(pred_positions)), dtype=torch.long))
        # 将 MLM 权重填充到最大预测位置数量，并转换为张量
        all_mlm_weights.append(torch.tensor([1.0] * len(mlm_pred_label_ids) + [0.0] * (max_num_mlm_preds - len(pred_positions)),
                                           dtype = torch.float32))
        # 将 MLM 预测标签填充到最大预测位置数量，并转换为张量
        all_mlm_labels.append(torch.tensor(mlm_pred_label_ids + [0] * (max_num_mlm_preds - len(mlm_pred_label_ids)),
                                           dtype=torch.long))
        # 记录 Next Sentence Prediction 的标签
        nsp_labels.append(torch.tensor(is_next, dtype=torch.long))
    # 返回所有填充后的输入数据
    return (all_token_ids, all_segments, valid_lens, all_pred_positions, 
            all_mlm_weights, all_mlm_labels, nsp_labels)

# The WikiText-2 dataset for pretraining BERT
class _WikiTextDataset(torch.utils.data.Dataset):
    def __init__(self, paragraphs, max_len):
        # 对段落进行分词，并构建词汇表
        paragraphs = [d2l.tokenize(
            paragraph, token='word') for paragraph in paragraphs]
        sentences = [sentence for paragraph in paragraphs 
                     for sentence in paragraph]
        self.vocab = d2l.Vocab(sentences, min_freq=5, reserved_tokens =[
            '<pad>', '<mask>', '<cls>', '<seq>'])
        examples = []
        # 遍历每个段落，生成 Next Sentence Prediction 任务的数据
        for paragraph in paragraphs:
            examples.extend(_get_nsp_data_from_paragraph(
                paragraph, paragraphs, self.vocab, max_len))
        # 遍历每个样本，生成 Masked Language Modeling 任务的数据，并对输入进行填充
        examples = [(_get_mlm_data_from_tokens(tokens, self.vocab)
                    + (segments, is_next))
                   for tokens, segments, is_next in examples]
        (self.all_token_ids, self.all_segments, self.valid_lens,
        self.all_pred_positions, self.all_mlm_weights,
        self.all_mlm_labels, self.nsp_labels) = _pad_bert_inputs(examples, max_len, self.vocab)   
        
    def __getitem__(self, idx):
        # 返回指定索引的数据
        return (self.all_token_ids[idx], self.all_segments[idx],
               self.valid_lens[idx], self.all_pred_positions[idx],
               self.all_mlm_weights[idx], self.all_mlm_labels[idx],
               self.nsp_labels[idx])
    
    def __len__(self):
        # 返回数据集的长度
        return len(self.all_token_ids)
# Download and WikiText-2 dataset and generate pretraining examples
def load_data_wiki(batch_size, max_len):
    """Load the WikiText-2 dataset. """
    # 获取用于加载数据的工作进程数
    num_workers = d2l.get_dataloader_workers()
    # 下载并解压 WikiText-2 数据集
    data_dir = d2l.download_extract('wikitext-2', 'wikitext-2')
    # 读取 WikiText-2 数据集中的段落
    paragraphs = _read_wiki(data_dir)
    # 构建 WikiText-2 数据集对象
    train_set = _WikiTextDataset(paragraphs, max_len)
    # 创建用于训练的数据迭代器
    train_iter = torch.utils.data.DataLoader(train_set, batch_size,
                                             shuffle=True, num_workers=0)  
    # 返回训练数据迭代器和词汇表
    return train_iter, train_set.vocab

# Print out the shapes of a minibatch of BERT pretraining examples
batch_size, max_len = 512, 64
# 加载 WikiText-2 数据集的训练数据迭代器和词汇表
train_iter, vocab = load_data_wiki(batch_size, max_len)
# 遍历训练数据迭代器，获取一个小批量的 BERT 预训练样本，并打印各项数据的形状
for (tokens_X, segments_X, valid_lens_x, pred_positions_X, mlm_weights_X,
    mlm_Y, nsp_y) in train_iter:
    print(tokens_X.shape, segments_X.shape, valid_lens_x.shape,
         pred_positions_X.shape, mlm_weights_X.shape, mlm_Y.shape,
         nsp_y.shape)
    break

torch.Size([512, 64]) torch.Size([512, 64]) torch.Size([512]) torch.Size([512, 10]) torch.Size([512, 10]) torch.Size([512, 10]) torch.Size([512])

# 打印词汇表的大小，即词汇表中不重复词汇的数量。
len(vocab)

20256

4. 预训练BERT

import torch
from torch import nn
from d2l import torch as d2l
# 设置了训练数据的批量大小 batch_size 和最大长度 max_len。
batch_size, max_len = 512, 64
# 调用 load_data_wiki 函数加载 WikiText-2 数据集，并返回训练数据迭代器 train_iter 和词汇表 vocab
train_iter, vocab = load_data_wiki(batch_size, max_len)
# A small BERT, using 2 layers, 128 hidden units, and 2 self-attention heads
# 创建一个小型的 BERT 模型，具有特定的配置
net = d2l.BERTModel(len(vocab), num_hiddens=128, norm_shape=[128], 
                    ffn_num_input=128, ffn_num_hiddens=256, num_heads=2,
                    num_layers=2, dropout=0.2, key_size=128, query_size=128,
                    value_size=128, hid_in_features=128, mlm_in_features=128,
                    nsp_in_features=128)
# 尝试使用所有可用的 GPU 设备
devices = d2l.try_all_gpus()
# 定义损失函数
loss = nn.CrossEntropyLoss()
# Computes the loss for both the masked language modeling and next sentence prediction tasks
# 计算遮蔽语言建模和下一个句子预测任务的损失
def _get_batch_loss_bert(net, loss, vocab_size, tokens_X,
                         segments_X, valid_lens_X,
                         pred_positions_X, mlm_weights_X,
                         mlm_Y, nsp_y):
    # 调用 BERT 模型进行前向传播，获取预测结果
    _, mlm_Y_hat, nsp_Y_hat = net(tokens_X, segments_X,
                                  valid_lens_x.reshape(-1),
                                  pred_positions_X)
    # 计算遮蔽语言建模任务的损失
    mlm_l = loss(mlm_Y_hat.reshape(-1, vocab_size), mlm_Y.reshape(-1)) * mlm_weights_X.reshape(-1, 1)     
    mlm_l = mlm_l.sum() / (mlm_weights_X.sum() + 1e-8)
    # 计算下一个句子预测任务的损失
    nsp_l = loss(nsp_Y_hat, nsp_y)
    # 总损失为遮蔽语言建模损失和下一个句子预测损失的和
    l = mlm_l + nsp_l
    return mlm_l, nsp_l, l
# Pretrain BERT(net) on the WikiText-2(train_iter) dataset
# 在 WikiText-2 数据集（train_iter）上对 BERT 模型（net）进行预训练
def train_bert(train_iter, net, loss, vocab_size, device, num_steps):
    # 将模型放到设备上并使用 DataParallel 进行多 GPU 训练
    net = nn.DataParallel(net, device_ids=devices).to(devices[0])
    # 定义优化器和学习率
    trainer = torch.optim.Adam(net.parameters(), lr=1e-3)
    # 初始化步数和计时器
    step, timer = 0, d2l.Timer()
    # 创建动画绘图器
    animator = d2l.Animator(xlabel='step', ylabel='loss',
                           xlim=[1, num_steps], legend=['mlm', 'nsp'])
    # 初始化指标累加器
    metric = d2l.Accumulator(4)
    # 标志位，判断是否达到指定步数
    num_steps_reached = False
    while step < num_steps and not num_steps_reached:
        for tokens_X, segments_X, valid_lens_x, pred_positions_X, mlm_weights_X, mlm_Y, nsp_y in train_iter:
            # 将数据移动到设备上
            tokens_X = tokens_X.to(devices[0])
            segments_X = segments_X.to(device[0])
            valid_lens_x = valid_lens_x.to(device[0])
            pred_positions_X = pred_positions_X.to(devices[0])
            mlm_weights_X = mlm_weights_X.to(devices[0])
            mlm_Y, nsp_y = mlm_Y.to(devices[0]), nsp_y.to(devices[0])
            # 梯度清零
            trainer.zero_grad()
            # 计时开始
            timer.start()
            mlm_l, nsp_l, l = _get_batch_loss_bert(net, loss, vocab_size, tokens_X, segments_X, valid_lens_x,
                                                   pred_positions_X, mlm_weights_X, mlm_Y, nsp_y)
            # 计算损失并进行反向传播和参数更新
            l.backward()
            trainer.step()
            # 累加指标和记录时间
            metric.add(mlm_l, nsp_l, tokens_X.shape[0], 1)
            timer.stop()
            # 绘制动画图像
            animator.add(step + 1, (metric[0] / metric[3], metric[1] / metric[3]))    
            step += 1
            # 如果达到指定步数，设置标志位并跳出循环
            if step == num_steps:
                num_steps_reached = True
                break
                
    # 打印最终结果和性能指标
    print(f'MLM loss {metric[0] / metric[3]:.3f}, '
          f'NSP loss {metric[1] / metric[3]:.3f}')
    print(f'{metric[2] / timer.sum():.1f} sentensece pairs/sec on '
          f'{str(devices)}')

# 调用函数进行 BERT 模型的预训练
train_bert(train_iter, net, loss, len(vocab), devices, 50)

MLM loss 6.008, NSP loss 0.698
2674.5 sentensece pairs/sec on [device(type='cuda', index=0)]

# Representing Text with BERT
# 使用 BERT 表示文本
def get_bert_encoding(net, tokens_a, tokens_b=None):
    # 获取 tokens 和 segments
    tokens, segments = d2l.get_tokens_and_segments(tokens_a, tokens_b)
    # 将 tokens 转换为 token_ids，并添加批次维度
    token_ids = torch.tensor(vocab[tokens], device=devices[0]).unsqueeze(0)   
    # 将 segments 转换为 tensor，并添加批次维度
    segments = torch.tensor(segments, device=devices[0]).unsqueeze(0)
    # 计算有效长度并添加批次维度
    valid_len = torch.tensor(len(tokens), device=devices[0]).unsqueeze(0)
    # 使用 BERT 进行编码
    encoded_X, _, _ = net(token_ids, segments, valid_len)
    return encoded_X
# Consider the sentence "a crane is flying"
# 考虑句子 "a crane is flying"
tokens_a = ['a', 'crane', 'is', 'flying']
# 使用 get_bert_encoding 函数对句子进行编码
encoded_text = get_bert_encoding(net, tokens_a)
# 提取编码后的句子的 CLS 标记
encoded_text_cls = encoded_text[:, 0, :]
# 提取编码后的句子中 "crane" 的表示
encoded_text_crane = encoded_text[:, 2, :]
# 输出编码后的句子的形状、CLS 标记的形状以及 "crane" 的前三个表示值
encoded_text.shape, encoded_text_cls.shape, encoded_text_crane[0][:3]
(torch.Size([1, 6, 128]),
 torch.Size([1, 128]),
 tensor([-0.4110,  1.2578, -0.4897], device='cuda:0', grad_fn=<SliceBackward0>))
# Now consider a sentence pair "a crane driver came" and "he just left"
# 现在考虑句子对 "a crane driver came" 和 "he just left"
tokens_a, tokens_b = ['a', 'crane', 'driver', 'came'], ['he', 'just', 'left']  
# 使用 get_bert_encoding 函数对句子对进行编码
encoded_pair = get_bert_encoding(net, tokens_a, tokens_b)
# 提取编码后的句子对的 CLS 标记
encoded_pair_cls = encoded_pair[:, 0, :]
# 提取编码后的句子对中 "crane" 的表示
encoded_pair_crane = encoded_pair[:, 2, :]
# 输出编码后的句子对的形状、CLS 标记的形状以及 "crane" 的前三个表示值
encoded_pair.shape, encoded_pair_cls.shape, encoded_pair_crane[0][:3]

(torch.Size([1, 10, 128]),
 torch.Size([1, 128]),
 tensor([-0.4008,  1.2771, -0.6009], device='cuda:0', grad_fn=<SliceBackward0>))