第TR5周：Transformer实战：文本分类

• 🍨 本文为 🔗365天深度学习训练营中的学习记录博客
• 🍖 原作者： K同学啊

一、准备工作

1.环境搭建

import torch,torchvision #注意是双下划线
print(torch.__version__)
print(torchvision.__version__)

1.10.0+cu113
0.11.1+cu113

2.加载数据

import torch
import torch.nn as nn
import torchvision
from torchvision import transforms,datasets
import os,PIL,pathlib,warnings
warnings.filterwarnings("ignore")#忽略警告信息
# win10系统
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device

device(type='cuda')

import pandas as pd
#加载自定义中文数据
train_data = pd.read_csv('F:/jupyter lab/DL-100-days/datasets/TR5/train.csv', sep='\t', header=None)
train_data.head()

| 0 | 1 |
| 0 | 还有双鸭山到淮阴的汽车票吗13号的 | Travel-Query |
| 1 | 从这里怎么回家 | Travel-Query |
| 2 | 随便播放一首专辑阁楼里的佛里的歌 | Music-Play |
| 3 | 给看一下墓王之王嘛 | FilmTele-Play |

4	我想看挑战两把s686打突变团竞的游戏视频	Video-Play

#构造数据集迭代器
def coustom_data_iter(texts,labels):
    for x,y in zip(texts,labels):
        yield x,y
        
train_iter = coustom_data_iter(train_data[0].values[:], train_data[1].values[:])

二、数据预处理

1.构建词典

from torchtext.data.utils import get_tokenizer
from torchtext.vocab import build_vocab_from_iterator
import jieba
#中文分词方法
tokenizer =jieba.lcut
def yield_tokens(data_iter):
    for text,_ in data_iter:
        yield tokenizer(text)
        
vocab = build_vocab_from_iterator(yield_tokens(train_iter), specials=["<unk>"])
vocab.set_default_index(vocab["<unk>"])# 设置默认索引，如果找不到单词，则会选择默认索引

Building prefix dict from the default dictionary ...
Dumping model to file cache C:\Users\LJH\AppData\Local\Temp\jieba.cache
Loading model cost 0.408 seconds.
Prefix dict has been built successfully.

vocab(['我','想','看','和平','精英','上','战神','必备','技巧','的','游戏','视频'])

2, 10, 13, 973, 1079, 146, 7724, 7574, 7793, 1, 186, 28

label_name = list(set(train_data[1].values[:]))
print(label_name)

['FilmTele-Play', 'Music-Play', 'HomeAppliance-Control', 'Audio-Play', 'Alarm-Update', 'Other', 'Video-Play', 'Calendar-Query', 'TVProgram-Play', 'Weather-Query', 'Radio-Listen', 'Travel-Query']

text_pipeline =lambda x:vocab(tokenizer(x))
label_pipeline = lambda x:label_name.index(x)
print(text_pipeline('我想看和平精英上战神必备技巧的游戏视频'))
print(label_pipeline('Video-Play'))

[2, 10, 13, 973, 1079, 146, 7724, 7574, 7793, 1, 186, 28]
6

2.生成数据批次和迭代器

from torch.utils.data import DataLoader
def collate_batch(batch):
    label_list, text_list = [], []
    
    for text, label in batch:
        label_list.append(label_pipeline(label))
        processed_text = torch.tensor(text_pipeline(text), dtype=torch.int64)
        text_list.append(processed_text)
    
    # pad_sequence 默认 pad 到最长句子长度
    text_list = pad_sequence(text_list, batch_first=True)  # shape: [B, L]
    label_list = torch.tensor(label_list, dtype=torch.int64)

    return text_list.to(device), label_list.to(device)

3.构建数据集

from torch.utils.data.dataset import random_split
from torchtext.data.functional import to_map_style_dataset

BATCH_SIZE =4

train_iter = coustom_data_iter(train_data[0].values[:], train_data[1].values[:]) 
train_dataset = to_map_style_dataset(train_iter)

split_train_,split_valid_ =random_split(train_dataset,
                                        [int(len(train_dataset)*0.8),int(len(train_dataset)*0.2)])

train_dataloader=DataLoader(split_train_,batch_size=BATCH_SIZE,
                            shuffle=True,collate_fn=collate_batch)

valid_dataloader = DataLoader(split_valid_ ,batch_size=BATCH_SIZE,
                              shuffle=True,collate_fn=collate_batch)

三、模型构建

1.定义位置编码函数

import math,os,torch
from torch.nn.utils.rnn import pad_sequence
class PositionalEncoding(nn.Module):
    def __init__ (self,embed_dim,max_len=500):
        
        super(PositionalEncoding,self).__init__()
        # 创建一个大小为[max_len,embed_dim]的零张量
        pe =torch.zeros(max_len,embed_dim)
        #创建一个形状为[maxlen,1]的位置索引张量
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        
        div_term = torch.exp(torch.arange(0, embed_dim, 2).float()*(-math.log(100.0)/ embed_dim))
        
        pe[:,0::2]=torch.sin(position*div_term)#计算 PE(pOs，2i)
        pe[:,1::2]= torch.cos(position*div_term)#计算 PE(pos，2i+1)
        pe =pe.unsqueeze(0).transpose(0,1)
        
        #将位置编码张量注册为模型的缓冲区,参数不参与梯度下降，保存model的时候会将其保存下来
        self.register_buffer('pe',pe)
        
    def forward(self,x):
        #将位置编码添加到输入张量中，注意位置编码的形状
        x=x+ self.pe[:x.size(0)]
        return x

2.定义Transformer模型

from tempfile import TemporaryDirectory
from typing import Tuple
from torch import nn,Tensor
from torch.nn import TransformerEncoder, TransformerEncoderLayer
from torch.utils.data import dataset

class TransformerModel(nn.Module):
    def __init__(self, vocab_size, embed_dim, num_class, nhead=4, d_hid=256, nlayers=2, dropout=0.2):
        super().__init__()
        self.embedding = nn.Embedding(vocab_size, embed_dim)
        self.pos_encoder = PositionalEncoding(embed_dim)
        encoder_layer = TransformerEncoderLayer(embed_dim, nhead, d_hid, dropout)
        self.transformer_encoder = TransformerEncoder(encoder_layer, nlayers)
        self.linear = nn.Linear(embed_dim, num_class)
    
    def forward(self, src, src_mask=None):
        src = self.embedding(src)                     # [B, L, D]
        src = self.pos_encoder(src.transpose(0, 1))   # [L, B, D]
        output = self.transformer_encoder(src, src_mask)  # [L, B, D]
        output = output.mean(dim=0)                   # [B, D]
        return self.linear(output)                    # [B, num_class]

3.初始化模型

vocab_size = len(vocab)#词汇表的大小
embed_dim = 64#嵌入维度
num_class= len(label_name)

#创建 Transformer 模型，并将其移动到设备上
model =TransformerModel(vocab_size,
                        embed_dim,
                        num_class).to(device)

4.定义训练函数

import time
def train(dataloader):
    model.train()#切换为训练模式
    total_acc,train_loss,total_count=0,0,0
    log_interval=300
    start_time=time.time()
    
    for idx,(text,label)in enumerate(dataloader):
        predicted_label=model(text)
        optimizer.zero_grad()            # grad属性归零

        loss = criterion(predicted_label,label)#计算网络输出和真实值之间的差距，label为真实值
        loss.backward()#反向传播
        optimizer.step()#每一步自动更新
        
        #记录acc与loss
        total_acc+=(predicted_label.argmax(1)== label).sum().item()
        train_loss+= loss.item()
        total_count += label.size(0)
        
        if idx % log_interval ==0 and idx >0:
            elapsed =time.time()-start_time
            print('|epoch{:1d}{:4d}/{:4d} batches'
                  '|train_acc {:4.3f} train_loss {:4.5f}'.format(epoch, idx, len(dataloader),
                                                total_acc/total_count,train_loss/total_count))
            total_acc,train_loss,total_count=0,0,0
            start_time =time.time()

5.定义评估函数

def evaluate(dataloader):
    model.eval()
    total_acc, val_loss, total_count = 0, 0, 0
    
    with torch.no_grad():
        for idx, (text, label) in enumerate(dataloader):
            predicted_label = model(text)
            loss = criterion(predicted_label, label)
            total_acc += (predicted_label.argmax(1) == label).sum().item()
            val_loss += loss.item()
            total_count += label.size(0)

    return total_acc / total_count, val_loss / total_count

四、训练模型

1.模型训练

#超参数
EPOCHS = 15
criterion =torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)

for epoch in range(1,EPOCHS + 1):
    epoch_start_time =time.time()
    train(train_dataloader)
    val_acc,val_loss =evaluate(valid_dataloader)
    
    #获取当前的学习率
    lr = optimizer.state_dict()['param_groups'][0]['lr']
    
    print('_'* 69)
    print('|epoch {:1d}|time:{:4.2f}s|'
          'valid_acc {:4.3f} valid_loss {:4.3f}| lr {:4.6f}'.format(epoch,time.time()-epoch_start_time,
                                                                    val_acc,val_loss,lr))
    print('_'* 69)

|epoch1 300/2420 batches|train_acc 0.348 train_loss 0.47728
|epoch1 600/2420 batches|train_acc 0.579 train_loss 0.32555
|epoch1 900/2420 batches|train_acc 0.667 train_loss 0.26298
|epoch11200/2420 batches|train_acc 0.718 train_loss 0.23648
|epoch11500/2420 batches|train_acc 0.758 train_loss 0.20101
|epoch11800/2420 batches|train_acc 0.792 train_loss 0.17321
|epoch12100/2420 batches|train_acc 0.797 train_loss 0.17088
|epoch12400/2420 batches|train_acc 0.797 train_loss 0.16852
_____________________________________________________________________
|epoch 1|time:33.86s|valid_acc 0.833 valid_loss 0.146| lr 0.001000
_____________________________________________________________________

.........

|epoch 14|time:36.07s|valid_acc 0.878 valid_loss 0.181| lr 0.001000
_____________________________________________________________________
|epoch15 300/2420 batches|train_acc 0.980 train_loss 0.01657
|epoch15 600/2420 batches|train_acc 0.975 train_loss 0.02135
|epoch15 900/2420 batches|train_acc 0.988 train_loss 0.01375
|epoch151200/2420 batches|train_acc 0.977 train_loss 0.01983
|epoch151500/2420 batches|train_acc 0.982 train_loss 0.01508
|epoch151800/2420 batches|train_acc 0.978 train_loss 0.01967
|epoch152100/2420 batches|train_acc 0.977 train_loss 0.01861
|epoch152400/2420 batches|train_acc 0.965 train_loss 0.02392
_____________________________________________________________________
|epoch 15|time:33.12s|valid_acc 0.879 valid_loss 0.177| lr 0.001000
_____________________________________________________________________

2.模型评估

test_acc,test_loss =evaluate(valid_dataloader)
print('模型准确率为:{:5.4f}'.format(test_acc))

模型准确率为:0.8785

五、学习心得

1.这段代码实现了一个基于 Transformer 的文本分类模型，通过对输入文本进行分词、编码和 padding，构建顺序保留的词向量序列，并结合位置编码和多层自注意力机制，有效捕捉句子中的上下文语义关系。

2.通过将nn.EmbeddingBag()：平均池化所有 token 向量变为nn.Embedding() + pad_sequence() 保留了每个 token 的顺序，能够通过 PositionalEncoding 和 Transformer 捕捉句法依赖。EmbeddingBag 强行"压平"文本，等于让模型看不到句子的结构。

3.output = output.mean(dim=0) # [batch, embed_dim]全局平均池化每句话，给分类器一个稳健的语义表征向量。

4.优化器变成Adam，自带梯度调整机制，更适合 NLP 和 Transformer 这类参数多、初始不稳定的网络。