以下是一个基于 PaddleNLP 的文本分类项目，按照标准工程结构组织，并包含测试数据集和完整流程。这个示例使用ERNIE模型处理IMDB电影评论情感分析任务。
项目工程结构
ernie_sentiment_analysis/
├── data/ # 数据集目录
│ ├── train.csv # 训练数据
│ ├── dev.csv # 验证数据
│ └── test.csv # 测试数据
├── configs/ # 配置文件
│ └── train_config.json # 训练参数配置
├── src/ # 源代码
│ ├── data_loader.py # 数据加载与处理
│ ├── model.py # 模型定义
│ ├── train.py # 训练脚本
│ ├── evaluate.py # 评估脚本
│ └── predict.py # 预测脚本
├── utils/ # 工具函数
│ ├── logger.py # 日志工具
│ └── metrics.py # 评估指标
├── output/ # 模型输出目录
│ └── ernie_model/ # 保存的模型文件
├── inference/ # 推理模型
│ └── model/ # 导出的推理模型
├── requirements.txt # 依赖包
└── README.md # 项目说明测试数据集示例
data/test.csv（电影评论情感分析）：
csv
text,label
This movie is really amazing! I love it.,1
The plot is so boring and the acting is terrible.,0
Best film I've seen this year. Highly recommended.,1
Waste of time and money. Avoid this film.,0核心代码实现
1. 数据加载与处理 (src/data_loader.py)
python
import pandas as pd
import paddle
from paddlenlp.datasets import load_dataset
from paddlenlp.transformers import ErnieTokenizer
from paddlenlp.data import Stack, Tuple, Pad
def read_csv(data_path):
df = pd.read_csv(data_path)
for _, row in df.iterrows():
yield {'text': row['text'], 'label': row['label']}
def get_dataloader(data_path, tokenizer, batch_size=32, max_seq_len=128, shuffle=False):
# 加载数据集
dataset = load_dataset(read_csv, data_path=data_path, lazy=False)
# 数据处理函数
def convert_example(example):
encoded_inputs = tokenizer(
text=example['text'],
max_seq_len=max_seq_len,
pad_to_max_seq_len=True
)
return {
'input_ids': encoded_inputs['input_ids'],
'token_type_ids': encoded_inputs['token_type_ids'],
'labels': example['label']
}
# 转换数据集
dataset = dataset.map(convert_example)
# 批量处理
batchify_fn = lambda samples, fn=Tuple(
Pad(axis=0, pad_val=tokenizer.pad_token_id), # input_ids
Pad(axis=0, pad_val=tokenizer.pad_token_type_id), # token_type_ids
Stack(dtype="int64") # labels
): fn(samples)
# 创建数据加载器
dataloader = paddle.io.DataLoader(
dataset=dataset,
batch_size=batch_size,
shuffle=shuffle,
collate_fn=batchify_fn
)
return dataloader2. 模型定义 (src/model.py)
python
from paddlenlp.transformers import ErnieForSequenceClassification
def create_model(num_classes, pretrained_model="ernie-1.0"):
model = ErnieForSequenceClassification.from_pretrained(
pretrained_model,
num_classes=num_classes
)
return model3. 训练脚本 (src/train.py)
python
import os
import json
import paddle
from paddlenlp.transformers import ErnieTokenizer
from data_loader import get_dataloader
from model import create_model
from utils.logger import setup_logger
from utils.metrics import compute_metrics
# 加载配置
with open('../configs/train_config.json', 'r') as f:
config = json.load(f)
# 设置日志
logger = setup_logger('train')
# 初始化设备
device = paddle.set_device('gpu' if paddle.is_available() else 'cpu')
# 加载分词器和模型
tokenizer = ErnieTokenizer.from_pretrained(config['pretrained_model'])
model = create_model(num_classes=2, pretrained_model=config['pretrained_model'])
model.to(device)
# 数据加载
train_dataloader = get_dataloader(
data_path='../data/train.csv',
tokenizer=tokenizer,
batch_size=config['batch_size'],
max_seq_len=config['max_seq_len'],
shuffle=True
)
dev_dataloader = get_dataloader(
data_path='../data/dev.csv',
tokenizer=tokenizer,
batch_size=config['batch_size'],
max_seq_len=config['max_seq_len']
)
# 优化器和损失函数
optimizer = paddle.optimizer.AdamW(
learning_rate=config['learning_rate'],
parameters=model.parameters()
)
criterion = paddle.nn.CrossEntropyLoss()
# 训练循环
for epoch in range(config['epochs']):
model.train()
total_loss = 0
for batch in train_dataloader:
input_ids, token_type_ids, labels = batch
input_ids = input_ids.to(device)
token_type_ids = token_type_ids.to(device)
labels = labels.to(device)
logits = model(input_ids, token_type_ids)
loss = criterion(logits, labels)
loss.backward()
optimizer.step()
optimizer.clear_grad()
total_loss += loss.item()
# 验证
model.eval()
predictions, labels = [], []
with paddle.no_grad():
for batch in dev_dataloader:
input_ids, token_type_ids, label = batch
input_ids = input_ids.to(device)
token_type_ids = token_type_ids.to(device)
logits = model(input_ids, token_type_ids)
pred = paddle.argmax(logits, axis=1)
predictions.extend(pred.cpu().numpy())
labels.extend(label.cpu().numpy())
metrics = compute_metrics(predictions, labels)
logger.info(f'Epoch [{epoch+1}/{config["epochs"]}]')
logger.info(f'Train Loss: {total_loss/len(train_dataloader):.4f}')
logger.info(f'Dev Metrics: {metrics}')
# 保存模型
model.save_pretrained(os.path.join('../output/ernie_model', f'epoch_{epoch+1}'))
tokenizer.save_pretrained(os.path.join('../output/ernie_model', f'epoch_{epoch+1}'))4. 评估脚本 (src/evaluate.py)
python
import paddle
from paddlenlp.transformers import ErnieTokenizer
from data_loader import get_dataloader
from model import create_model
from utils.metrics import compute_metrics
# 加载模型和分词器
model = create_model(num_classes=2)
model.set_state_dict(paddle.load('../output/ernie_model/best_model/model_state.pdparams'))
tokenizer = ErnieTokenizer.from_pretrained('../output/ernie_model/best_model')
# 加载测试数据
test_dataloader = get_dataloader(
data_path='../data/test.csv',
tokenizer=tokenizer,
batch_size=32
)
# 评估
model.eval()
predictions, labels = [], []
with paddle.no_grad():
for batch in test_dataloader:
input_ids, token_type_ids, label = batch
logits = model(input_ids, token_type_ids)
pred = paddle.argmax(logits, axis=1)
predictions.extend(pred.numpy())
labels.extend(label.numpy())
# 计算指标
metrics = compute_metrics(predictions, labels)
print(f'Test Metrics: {metrics}')5. 预测脚本 (src/predict.py)
python
import paddle
from paddlenlp.transformers import ErnieTokenizer
from model import create_model
def predict(text, model, tokenizer, max_seq_len=128):
model.eval()
encoded_inputs = tokenizer(
text=text,
max_seq_len=max_seq_len,
pad_to_max_seq_len=True,
return_tensors='pd'
)
with paddle.no_grad():
logits = model(encoded_inputs['input_ids'], encoded_inputs['token_type_ids'])
probs = paddle.nn.functional.softmax(logits, axis=1)
pred = paddle.argmax(probs, axis=1).item()
confidence = probs[0][pred].item()
sentiment = 'Positive' if pred == 1 else 'Negative'
return {
'text': text,
'sentiment': sentiment,
'confidence': confidence
}
# 加载模型和分词器
model = create_model(num_classes=2)
model.set_state_dict(paddle.load('../output/ernie_model/best_model/model_state.pdparams'))
tokenizer = ErnieTokenizer.from_pretrained('../output/ernie_model/best_model')
# 示例预测
text = "This movie is absolutely fantastic! I can't wait to watch it again."
result = predict(text, model, tokenizer)
print(f"预测结果: {result}")配置文件示例 (configs/train_config.json)
json
{
"pretrained_model": "ernie-1.0",
"batch_size": 32,
"max_seq_len": 128,
"learning_rate": 2e-5,
"epochs": 3,
"save_dir": "../output/ernie_model"
}测试数据集生成脚本
python
import pandas as pd
# 示例数据
data = {
'text': [
"This movie is really amazing! I love it.",
"The plot is so boring and the acting is terrible.",
"Best film I've seen this year. Highly recommended.",
"Waste of time and money. Avoid this film.",
"The special effects are incredible, but the story is weak.",
"I couldn't stop laughing. Great comedy!",
"Terrible. Don't waste your time.",
"A masterpiece. Definitely worth watching."
],
'label': [1, 0, 1, 0, 0, 1, 0, 1]
}
# 创建DataFrame
df = pd.DataFrame(data)
# 划分训练集、验证集和测试集
train_df = df.iloc[:5]
dev_df = df.iloc[5:7]
test_df = df.iloc[7:]
# 保存到CSV
train_df.to_csv('data/train.csv', index=False)
dev_df.to_csv('data/dev.csv', index=False)
test_df.to_csv('data/test.csv', index=False)
print("数据集生成完成！")使用说明
-
安装依赖：
bashpip install -r requirements.txt -
训练模型：
bashpython src/train.py -
评估模型：
bashpython src/evaluate.py -
预测新文本：
bashpython src/predict.py
扩展建议
- 添加更多任务：如命名实体识别、文本生成等。
- 增加模型选择：支持BERT、RoBERTa等不同预训练模型。
- 添加早停和模型选择：根据验证集性能自动选择最佳模型。
- 添加超参数调优：集成optuna等工具进行超参数搜索。
这个项目结构清晰，模块化程度高，便于扩展和维护，可以作为NLP项目的基础框架。