PaddleNLP 的文本分类项目

以下是一个基于 PaddleNLP 的文本分类项目，按照标准工程结构组织，并包含测试数据集和完整流程。这个示例使用ERNIE模型处理IMDB电影评论情感分析任务。

项目工程结构

ernie_sentiment_analysis/
├── data/                  # 数据集目录
│   ├── train.csv          # 训练数据
│   ├── dev.csv            # 验证数据
│   └── test.csv           # 测试数据
├── configs/               # 配置文件
│   └── train_config.json  # 训练参数配置
├── src/                   # 源代码
│   ├── data_loader.py     # 数据加载与处理
│   ├── model.py           # 模型定义
│   ├── train.py           # 训练脚本
│   ├── evaluate.py        # 评估脚本
│   └── predict.py         # 预测脚本
├── utils/                 # 工具函数
│   ├── logger.py          # 日志工具
│   └── metrics.py         # 评估指标
├── output/                # 模型输出目录
│   └── ernie_model/       # 保存的模型文件
├── inference/             # 推理模型
│   └── model/             # 导出的推理模型
├── requirements.txt       # 依赖包
└── README.md              # 项目说明

测试数据集示例

data/test.csv（电影评论情感分析）：

text,label
This movie is really amazing! I love it.,1
The plot is so boring and the acting is terrible.,0
Best film I've seen this year. Highly recommended.,1
Waste of time and money. Avoid this film.,0

核心代码实现

1. 数据加载与处理 (src/data_loader.py)

import pandas as pd
import paddle
from paddlenlp.datasets import load_dataset
from paddlenlp.transformers import ErnieTokenizer
from paddlenlp.data import Stack, Tuple, Pad

def read_csv(data_path):
    df = pd.read_csv(data_path)
    for _, row in df.iterrows():
        yield {'text': row['text'], 'label': row['label']}

def get_dataloader(data_path, tokenizer, batch_size=32, max_seq_len=128, shuffle=False):
    # 加载数据集
    dataset = load_dataset(read_csv, data_path=data_path, lazy=False)
    
    # 数据处理函数
    def convert_example(example):
        encoded_inputs = tokenizer(
            text=example['text'],
            max_seq_len=max_seq_len,
            pad_to_max_seq_len=True
        )
        return {
            'input_ids': encoded_inputs['input_ids'],
            'token_type_ids': encoded_inputs['token_type_ids'],
            'labels': example['label']
        }
    
    # 转换数据集
    dataset = dataset.map(convert_example)
    
    # 批量处理
    batchify_fn = lambda samples, fn=Tuple(
        Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input_ids
        Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token_type_ids
        Stack(dtype="int64")  # labels
    ): fn(samples)
    
    # 创建数据加载器
    dataloader = paddle.io.DataLoader(
        dataset=dataset,
        batch_size=batch_size,
        shuffle=shuffle,
        collate_fn=batchify_fn
    )
    
    return dataloader

2. 模型定义 (src/model.py)

from paddlenlp.transformers import ErnieForSequenceClassification

def create_model(num_classes, pretrained_model="ernie-1.0"):
    model = ErnieForSequenceClassification.from_pretrained(
        pretrained_model,
        num_classes=num_classes
    )
    return model

3. 训练脚本 (src/train.py)

import os
import json
import paddle
from paddlenlp.transformers import ErnieTokenizer
from data_loader import get_dataloader
from model import create_model
from utils.logger import setup_logger
from utils.metrics import compute_metrics

# 加载配置
with open('../configs/train_config.json', 'r') as f:
    config = json.load(f)

# 设置日志
logger = setup_logger('train')

# 初始化设备
device = paddle.set_device('gpu' if paddle.is_available() else 'cpu')

# 加载分词器和模型
tokenizer = ErnieTokenizer.from_pretrained(config['pretrained_model'])
model = create_model(num_classes=2, pretrained_model=config['pretrained_model'])
model.to(device)

# 数据加载
train_dataloader = get_dataloader(
    data_path='../data/train.csv',
    tokenizer=tokenizer,
    batch_size=config['batch_size'],
    max_seq_len=config['max_seq_len'],
    shuffle=True
)

dev_dataloader = get_dataloader(
    data_path='../data/dev.csv',
    tokenizer=tokenizer,
    batch_size=config['batch_size'],
    max_seq_len=config['max_seq_len']
)

# 优化器和损失函数
optimizer = paddle.optimizer.AdamW(
    learning_rate=config['learning_rate'],
    parameters=model.parameters()
)
criterion = paddle.nn.CrossEntropyLoss()

# 训练循环
for epoch in range(config['epochs']):
    model.train()
    total_loss = 0
    for batch in train_dataloader:
        input_ids, token_type_ids, labels = batch
        input_ids = input_ids.to(device)
        token_type_ids = token_type_ids.to(device)
        labels = labels.to(device)
        
        logits = model(input_ids, token_type_ids)
        loss = criterion(logits, labels)
        loss.backward()
        optimizer.step()
        optimizer.clear_grad()
        
        total_loss += loss.item()
    
    # 验证
    model.eval()
    predictions, labels = [], []
    with paddle.no_grad():
        for batch in dev_dataloader:
            input_ids, token_type_ids, label = batch
            input_ids = input_ids.to(device)
            token_type_ids = token_type_ids.to(device)
            
            logits = model(input_ids, token_type_ids)
            pred = paddle.argmax(logits, axis=1)
            
            predictions.extend(pred.cpu().numpy())
            labels.extend(label.cpu().numpy())
    
    metrics = compute_metrics(predictions, labels)
    
    logger.info(f'Epoch [{epoch+1}/{config["epochs"]}]')
    logger.info(f'Train Loss: {total_loss/len(train_dataloader):.4f}')
    logger.info(f'Dev Metrics: {metrics}')
    
    # 保存模型
    model.save_pretrained(os.path.join('../output/ernie_model', f'epoch_{epoch+1}'))
    tokenizer.save_pretrained(os.path.join('../output/ernie_model', f'epoch_{epoch+1}'))

4. 评估脚本 (src/evaluate.py)

import paddle
from paddlenlp.transformers import ErnieTokenizer
from data_loader import get_dataloader
from model import create_model
from utils.metrics import compute_metrics

# 加载模型和分词器
model = create_model(num_classes=2)
model.set_state_dict(paddle.load('../output/ernie_model/best_model/model_state.pdparams'))
tokenizer = ErnieTokenizer.from_pretrained('../output/ernie_model/best_model')

# 加载测试数据
test_dataloader = get_dataloader(
    data_path='../data/test.csv',
    tokenizer=tokenizer,
    batch_size=32
)

# 评估
model.eval()
predictions, labels = [], []
with paddle.no_grad():
    for batch in test_dataloader:
        input_ids, token_type_ids, label = batch
        logits = model(input_ids, token_type_ids)
        pred = paddle.argmax(logits, axis=1)
        
        predictions.extend(pred.numpy())
        labels.extend(label.numpy())

# 计算指标
metrics = compute_metrics(predictions, labels)
print(f'Test Metrics: {metrics}')

5. 预测脚本 (src/predict.py)

import paddle
from paddlenlp.transformers import ErnieTokenizer
from model import create_model

def predict(text, model, tokenizer, max_seq_len=128):
    model.eval()
    encoded_inputs = tokenizer(
        text=text,
        max_seq_len=max_seq_len,
        pad_to_max_seq_len=True,
        return_tensors='pd'
    )
    
    with paddle.no_grad():
        logits = model(encoded_inputs['input_ids'], encoded_inputs['token_type_ids'])
        probs = paddle.nn.functional.softmax(logits, axis=1)
        pred = paddle.argmax(probs, axis=1).item()
        confidence = probs[0][pred].item()
    
    sentiment = 'Positive' if pred == 1 else 'Negative'
    return {
        'text': text,
        'sentiment': sentiment,
        'confidence': confidence
    }

# 加载模型和分词器
model = create_model(num_classes=2)
model.set_state_dict(paddle.load('../output/ernie_model/best_model/model_state.pdparams'))
tokenizer = ErnieTokenizer.from_pretrained('../output/ernie_model/best_model')

# 示例预测
text = "This movie is absolutely fantastic! I can't wait to watch it again."
result = predict(text, model, tokenizer)
print(f"预测结果: {result}")

配置文件示例 (configs/train_config.json)

{
    "pretrained_model": "ernie-1.0",
    "batch_size": 32,
    "max_seq_len": 128,
    "learning_rate": 2e-5,
    "epochs": 3,
    "save_dir": "../output/ernie_model"
}

测试数据集生成脚本

import pandas as pd

# 示例数据
data = {
    'text': [
        "This movie is really amazing! I love it.",
        "The plot is so boring and the acting is terrible.",
        "Best film I've seen this year. Highly recommended.",
        "Waste of time and money. Avoid this film.",
        "The special effects are incredible, but the story is weak.",
        "I couldn't stop laughing. Great comedy!",
        "Terrible. Don't waste your time.",
        "A masterpiece. Definitely worth watching."
    ],
    'label': [1, 0, 1, 0, 0, 1, 0, 1]
}

# 创建DataFrame
df = pd.DataFrame(data)

# 划分训练集、验证集和测试集
train_df = df.iloc[:5]
dev_df = df.iloc[5:7]
test_df = df.iloc[7:]

# 保存到CSV
train_df.to_csv('data/train.csv', index=False)
dev_df.to_csv('data/dev.csv', index=False)
test_df.to_csv('data/test.csv', index=False)

print("数据集生成完成！")

使用说明

1. 安装依赖：

   pip install -r requirements.txt

2. 训练模型：

   python src/train.py

3. 评估模型：

   python src/evaluate.py

4. 预测新文本：

   python src/predict.py

扩展建议

1. 添加更多任务：如命名实体识别、文本生成等。
2. 增加模型选择：支持BERT、RoBERTa等不同预训练模型。
3. 添加早停和模型选择：根据验证集性能自动选择最佳模型。
4. 添加超参数调优：集成optuna等工具进行超参数搜索。

这个项目结构清晰，模块化程度高，便于扩展和维护，可以作为NLP项目的基础框架。