import os
from paddlenlp.data import DataCollatorWithPadding
import random
import numpy as np
import paddle
import json
from paddlenlp.utils.log import logger
from paddlenlp.transformers import (AutoModelForSequenceClassification, AutoTokenizer)
from paddlenlp.datasets import MapDataset
更新模型的dropout
def update_model_dropout(model, p=0.0):
model.base_model.embeddings.dropout.p = p
for i in range(len(model.base_model.encoder.layers)):
model.base_model.encoder.layers[i].dropout.p = p
model.base_model.encoder.layers[i].dropout1.p = p
model.base_model.encoder.layers[i].dropout2.p = p
负样本标题neg_title有一定概率和正样本标题title一样
所以对于新生成的样本还要过滤,把一样的过滤掉
def gen_pair(dataset, pool_size=100):
if len(dataset) < pool_size:
pool_size = len(dataset)
new_examples = [] # 新样本
pool = [] # 临时容器,存放样本对应的title
tmp_exmaples = [] # 临时容器,存放样本
for example in dataset:
label = example["label"]
这里是生成neg_title,label==0的本来就是neg_title
要生成也是正标题,但是正标题无法生成
if label == 0:
continue
tmp_exmaples.append(example)
pool.append(example["title"]) #池子存放的是样本对应的标题
if len(pool) >= pool_size: # 如果够批次了
np.random.shuffle(pool) # 随机刷新title顺序
#遍历临时容器中的每个样本
for idx, example in enumerate(tmp_exmaples):
设置neg_title为别人的title
example["neg_title"] = pool[idx]
把修改后的样本加入新样本集
new_examples.append(example)
tmp_exmaples = [] # 清空,以存放下个批次数据
pool = []
if len(pool)>0:
np.random.shuffle(pool)
for idx, example in enumerate(tmp_exmaples):
设置neg_title为别人的title
example["neg_title"] = pool[idx]
把修改后的样本加入新样本集
new_examples.append(example)
return MapDataset(new_examples)
def cal_md5(str):
str = str.decode("utf-8", "ignore").encode("utf-8", "ignore")
return hashlib.md5(str).hexdigest()
def create_dataloader(dataset,
batch_size=1,
batchify_fn=None,
trans_fn=None,
return_list=True,
mode='train'):
if trans_fn:
dataset = dataset.map(trans_fn)
shuffle = True if mode == 'train' else False
batch_sampler = paddle.io.BatchSampler(dataset,
batch_size=batch_size,
shuffle=shuffle)
if not return_list:
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn)
return paddle.io.DataLoader(dataset=dataset,
batch_sampler=batch_sampler,
collate_fn=batchify_fn,
return_list=return_list)
重复词策略
def word_repetition(input_ids, token_type_ids, dup_rate=0.32):
"""Word Repetition strategy."""
input_ids = input_ids.numpy().tolist()
token_type_ids = token_type_ids.numpy().tolist()
batch_size, seq_len = len(input_ids), len(input_ids[0])
repetitied_input_ids = [] # 用来装重复词后的批次ids
repetitied_token_type_ids = [] # 用来装重复词后的批次sids
rep_seq_len = seq_len # 用来设定重复词策略后的批次最大序列长度
for batch_id in range(batch_size):
cur_input_id = input_ids[batch_id]
actual_len = np.count_nonzero(cur_input_id) # 非填充token
dup_word_index = []
If sequence length is less than 5, skip it
if actual_len > 5:
重复长度是0--int(dup_rate * actual_len)之间的值
dup_len = random.randint(a=0, b=max(2, int(dup_rate * actual_len)))
刨除[CLS]和[SEP],随机采样dup_len个
dup_word_index = random.sample(list(range(1, actual_len - 1)), k=dup_len)
r_input_id = []
r_token_type_id = []
for idx, word_id in enumerate(cur_input_id):
插入重复单词,如果idx在dup_word_index中,idx从0开始,
在里面,说明被采样为要重复的token
if idx in dup_word_index:
r_input_id.append(word_id)
r_token_type_id.append(token_type_ids[batch_id][idx])
正常的token只会被添加一次,选中的重复词token会被添加两次
实现了重复词策略
r_input_id.append(word_id)
r_token_type_id.append(token_type_ids[batch_id][idx])
after_dup_len = len(r_input_id) # 重复词后的批次内单样本长度
repetitied_input_ids.append(r_input_id)
repetitied_token_type_ids.append(r_token_type_id)
更新rep_seq_len
if after_dup_len > rep_seq_len:
rep_seq_len = after_dup_len
填充批次数据到同一序列长度
for batch_id in range(batch_size):
after_dup_len = len(repetitied_input_ids[batch_id]) # 这个批次内第i个样本的序列长度
pad_len = rep_seq_len - after_dup_len # 要填充的长度
repetitied_input_ids[batch_id] += [0] * pad_len
repetitied_token_type_ids[batch_id] += [0] * pad_len
返回重复词策略后的数据
return paddle.to_tensor(repetitied_input_ids, dtype="int64"), paddle.to_tensor(
repetitied_token_type_ids, dtype="int64"
)
def set_seed(seed):
random.seed(seed)
np.random.seed(seed)
paddle.seed(seed)
def read_by_lines(path):
result = []
with open(path, "r", encoding="utf8") as f:
for line in f:
result.append(line.strip())
return result
def write_by_lines(path, data):
with open(path, "w", encoding="utf8") as f:
[f.write(d + "\n") for d in data]
def write_text(path,data):
with open(path,mode='w',encoding='utf-8') as fout:
for i in data:
fout.write('{}\n'.format(i.strip()))
import os
import hnswlib
from paddlenlp.utils.log import logger
import numpy as np
import paddle
import json
@paddle.no_grad()
def evaluate_glue(model, loss_fct, metric, data_loader):
model.eval()
metric.reset()
for batch in data_loader:
input_ids, labels = batch
logits = model(input_ids)
loss = loss_fct(logits,labels)
correct = metric.compute(logits,labels)
metric.update(correct)
res = metric.accumulate()
if isinstance(metric, AccuracyAndF1):
print(
"eval loss: %f, acc: %s, precision: %s, recall: %s, f1: %s, acc and f1: %s, "
% (loss.item(),*res))
elif isinstance(metric, Mcc):
print("eval loss: %f, mcc: %s, " % (loss.item(),res[0]))
elif isinstance(metric, PearsonAndSpearman):
print(
"eval loss: %f, pearson: %s, spearman: %s, pearson and spearman: %s, "
% (loss.item(),*res),
end='')
else:
print("eval loss: %f, acc: %s, " % (loss.item(), res))
metric.reset()
model.train()
return res[0] if isinstance(res,list) else res
@paddle.no_grad()
def evaluate_clue(model, loss_fct, metric, data_loader):
model.eval()
metric.reset()
for batch in data_loader:
labels = batch.pop("labels") # 弹出栈,labels
logits = model(**batch) # **自动拆包
loss = loss_fct(logits, labels)
correct = metric.compute(logits, labels) # 计算正确数
metric.update(correct)
res = metric.accumulate() # 评估结果
logger.info("eval loss: %f, acc: %s, " % (loss.item(), res))
metric.reset()
model.train()
return res
@paddle.no_grad()
def do_evaluate2(model, tokenizer, data_loader, label_normalize_dict):
model.eval()
total_num = 0
correct_num = 0
normed_labels = [
normalized_lable
for origin_lable, normalized_lable in label_normalize_dict.items()
]
label_length = len(normed_labels[0]) # 标签长度
for batch in data_loader:
src_ids, token_type_ids, masked_positions, masked_lm_labels = batch
[bs * label_length, vocab_size]
prediction_probs = model.predict(input_ids=src_ids,
token_type_ids=token_type_ids,
masked_positions=masked_positions)
batch_size = len(src_ids)
vocab_size = prediction_probs.shape[1]
prediction_probs: [batch_size, label_length, vocab_size]
prediction_probs = paddle.reshape(prediction_probs,
shape=[batch_size, -1,
vocab_size]).numpy()
[label_num, label_length]
label_ids = np.array(
[tokenizer(label)["input_ids"][1:-1] for label in normed_labels])
y_pred = np.ones(shape=[batch_size, len(label_ids)])
计算候选标签的联合分布。
for index in range(label_length):
y_pred *= prediction_probs[:, index, label_ids[:, index]]
Get max probs label's index
y_pred_index = np.argmax(y_pred, axis=-1)
y_true_index = []
for masked_lm_label in masked_lm_labels.numpy():
label_text = "".join(
tokenizer.convert_ids_to_tokens(list(masked_lm_label)))
label_index = normed_labels.index(label_text)
y_true_index.append(label_index)
y_true_index = np.array(y_true_index)
total_num += len(y_true_index)
correct_num += (y_true_index == y_pred_index).sum()
model.train()
return 100 * correct_num / total_num, total_num
@paddle.no_grad()
def do_evaluate_chid(model, tokenizer, data_loader, label_normalize_dict):
"""
FCLUE `chid` 数据集在评估时具有特殊性：输入槽中包含额外的 `candidate_label_ids`，
因此需要自定义评估函数。
"""
model.eval()
total_num = 0
correct_num = 0
normed_labels = [
normalized_lable
for origin_lable, normalized_lable in label_normalize_dict.items()
]
label_length = len(normed_labels[0])
for batch in data_loader:
src_ids, token_type_ids, masked_positions, masked_lm_labels, candidate_label_ids = batch
[bs * label_length, vocab_size]
prediction_probs = model.predict(input_ids=src_ids,
token_type_ids=token_type_ids,
masked_positions=masked_positions)
batch_size = len(src_ids)
vocab_size = prediction_probs.shape[1]
prediction_probs: [batch_size, label_lenght, vocab_size]
prediction_probs = paddle.reshape(prediction_probs,
shape=[batch_size, -1,
vocab_size]).numpy()
candidate_num = candidate_label_ids.shape[1]
[batch_size, candidate_num(7)]
y_pred = np.ones(shape=[batch_size, candidate_num])
for label_idx in range(candidate_num):
[bathc_size, label_length(4)]
single_candidate_label_ids = candidate_label_ids[:, label_idx, :]
Calculate joint distribution of candidate labels
for index in range(label_length):
[batch_size,]
slice_word_ids = single_candidate_label_ids[:, index].numpy()
batch_single_token_prob = []
for bs_index in range(batch_size):
[1, 1]
single_token_prob = prediction_probs[
bs_index, index, slice_word_ids[bs_index]]
batch_single_token_prob.append(single_token_prob)
y_pred[:, label_idx] *= np.array(batch_single_token_prob)
Get max probs label's index
y_pred_index = np.argmax(y_pred, axis=-1)
y_true_index = []
for index, masked_lm_label in enumerate(masked_lm_labels.numpy()):
[cantidate_num, label_length]
tmp_candidate_label_ids = candidate_label_ids[index, :, :]
for idx, label_ids in enumerate(tmp_candidate_label_ids.numpy()):
if np.equal(label_ids, masked_lm_label).all():
y_true_index.append(idx)
continue
y_true_index = np.array(y_true_index)
total_num += len(y_true_index)
correct_num += (y_true_index == y_pred_index).sum()
model.train()
return 100 * correct_num / total_num, total_num
@paddle.no_grad()
def do_evaluate(model, tokenizer, data_loader, task_label_description):
model.eval()
total_num = 0
correct_num = 0
class_num = len(task_label_description) # 15
[total_num * class_num, 2]
all_prediction_probs = []
[total_num * class_num]
all_labels = []
for batch in data_loader:
src_ids, token_type_ids, true_labels = batch
prediction_probs = model(input_ids=src_ids,
token_type_ids=token_type_ids).numpy()
all_prediction_probs.append(prediction_probs)
all_labels.append(true_labels.numpy())
all_labels = np.concatenate(all_labels, axis=0)
all_prediction_probs = np.concatenate(all_prediction_probs, axis=0)
all_prediction_probs = np.reshape(all_prediction_probs, (-1, class_num, 2))
(total_num,class_num,1),1是属于正样本对的分数,0是属于负样本对的分数
prediction_pos_probs = all_prediction_probs[:, :, 1]
prediction_pos_probs = np.reshape(prediction_pos_probs, (-1, class_num))
获取total_num中每个样本属于各个类别的预测类别,y_pred_index的值会是
0--class_num-1间的值,属于模型预测sentence1和sentence2中哪一个是正样本
y_pred_index = np.argmax(prediction_pos_probs, axis=-1)
每15个是同一个原样本和不同提示的样本对,这里只用每个原样本一个真实标签就行
idx % class_num == 0保证15个里只取一个索引,这个索引对应真实标签
y_true_index = np.array([
true_label_index for idx, true_label_index in enumerate(all_labels)
if idx % class_num == 0
])
total_num = len(y_true_index) # 总的评估样本数,sentence1
correct_num = (y_pred_index == y_true_index).sum() #预测对的样本数
model.train()
return 100 * correct_num / total_num, total_num
hnsw_max_elements=1000000
hnsw_ef=100
hnsw_m=100
def build_index(data_loader, model,output_emb_size):
index = hnswlib.Index(
space='ip',
dim=output_emb_size if output_emb_size > 0 else 768)
index.init_index(max_elements=hnsw_max_elements,
ef_construction=hnsw_ef,
M=hnsw_m)
index.set_ef(hnsw_ef)
index.set_num_threads(6)
logger.info("start build index..........")
all_embeddings = []
for text_embeddings in model.get_semantic_embedding(data_loader):
all_embeddings.append(text_embeddings.numpy())
all_embeddings = np.concatenate(all_embeddings, axis=0)
index.add_items(all_embeddings)
logger.info("Total index number:{}".format(index.get_current_count()))
return index
def write_recall_file(model,query_data_loader,final_index,text_list,
id2corpus,recall_result_file,recall_num=20):
query_embedding = model.get_semantic_embedding(query_data_loader)
with open(recall_result_file, 'w', encoding='utf-8') as f:
for batch_index, batch_query_embedding in enumerate(query_embedding):
recalled_idx, cosine_sims = final_index.knn_query(\
batch_query_embedding.numpy(),recall_num)
batch_size = len(cosine_sims)
for row_index in range(batch_size):
text_index = batch_size * batch_index + row_index # 对应query列表中的文本
#把原query,从索引库召回的50条语义相近索引,前两者的相似度写入召回文件
for idx, doc_idx in enumerate(recalled_idx[row_index]):
f.write("{}\t{}\t{}\n".format(
text_list[text_index]["text"], id2corpus[doc_idx],
1.0 - cosine_sims[row_index][idx]))
@paddle.no_grad()
def evaluate(model, corpus_data_loader, query_data_loader, recall_result_file,output_emb_size,
text_list,text2similar,id2corpus,recall_num=20,final_index=None):
model.eval()
def recall(rs, N=10):
recall_flags = [np.sum(r[:N]) for r in rs]#前N个有一个1就是1
return np.mean(recall_flags)#返回的是topK召回精确率
构建索引库，这个不能写外面，因为构建索引的模型每次都不一样,如果写外面,传进来,这在模型不训练只评估时还行
但是在训练中评估时就不行,因为不同模型构建的索引库不同,因为由他们提前的向量不同,你用新模型提取的向量
#去旧的索引库查找,肯定出问题.
if final_index is None:
final_index = build_index(corpus_data_loader,model,output_emb_size)
write_recall_file(model,query_data_loader,final_index,text_list,
id2corpus,recall_result_file,recall_num)
rs = []
with open(recall_result_file, 'r', encoding='utf-8') as f:
relevance_labels = []
for index, line in enumerate(f):
#用来保存一个文本的召回文本的标记，召回到相似文本标记为1
if index % recall_num == 0 and index != 0:
rs.append(relevance_labels)
relevance_labels = []# 够一个原query的召回就清空，之后存放下个query的召回
text, recalled_text, cosine_sim = line.rstrip().split("\t")#原文本，召回文本，距离
#召回是模拟用户不规则的query,去召回语料库中的title,如果召回的文本和相似文本对中query对应的
文本一样,那说明成功召回,设置当前召回标记1,不管多少召回,就只会有1个标记1的,但是算精度的时候,
只要成功,在召回k中成功召回,都是1召回精度衡量的是N次召回的平均成功几率,text2similar中query
对应的相似文本必须在corpus语料库里存在,否则就不可能成功召回,所以在评估前必须校准一下,query
对应的必须是title,如果不是,就得过滤掉，因为如果在corpus里找不到,就不可能召回,标记里全是0,
评估就失准
if text2similar[text] == recalled_text:# 成功召回，设置标记1,表示找到
relevance_labels.append(1)
else:
relevance_labels.append(0)# 不相同就设置0,表示没找到
recall_N = []# 召回精确率,N越小,召回精度越高越好
recall_nums = [1, 5, 10, 20]
for topN in recall_nums:# 遍历召回数
R = round(100 * recall(rs, N=topN), 3)
recall_N.append(R)
evaluate_result_file = os.path.join(recall_result_dir,
evaluate_result)
result = open(evaluate_result_file, 'a')
res = []
timestamp = time.strftime('%Y%m%d-%H%M%S', time.localtime())
res.append(timestamp)
for key, val in zip(recall_nums, recall_N):
print('recall@{}={}'.format(key, val))
res.append(str(val))
result.write('\t'.join(res) + '\n')
print(res)
model.train()
score=recall_N[0]*0.3+recall_N[1]*0.3+recall_N[2]*0.2+recall_N[3]*0.2
return score
def predict_rank_predict(model, data_loader):
all_probs = []
model.eval()
with paddle.no_grad():
for batch_data in data_loader:
input_ids, token_type_ids = batch_data
batch_prob = model.predict(input_ids=input_ids,#批次相似度
token_type_ids=token_type_ids).numpy()
all_probs.append(batch_prob)
if (len(all_probs) == 1): # 只预测一个批次的情况
all_probs = np.array(all_probs)
else:
all_probs = np.concatenate(all_probs, axis=0) # 合并
return all_probs
@paddle.no_grad()
def evaluate_rank_auc(model, metric, data_loader, phase="dev"):
model.eval()
metric.reset()
for idx, batch in enumerate(data_loader):
input_ids, token_type_ids, labels = batch
获取预测概率
pos_probs = model.predict(input_ids=input_ids,
token_type_ids=token_type_ids)
neg_probs = 1.0 - pos_probs # 负概率
预测(neg_probs, pos_probs)
preds = np.concatenate((neg_probs, pos_probs), axis=1)
metric.update(preds=preds, labels=labels)
auc=metric.accumulate()
print("eval_{} auc:{:.3}".format(phase,auc))
metric.reset()
model.train()
return auc
@paddle.no_grad()
def evaluate_seq_classification(model, criterion, metric, data_loader):
model.eval() # 评估模式
metric.reset() # 指标重置
losses = [] # 用来保存批次损失
for batch in data_loader:
input_ids,token_type_ids,labels=batch
logits = model(input_ids,token_type_ids)
loss = criterion(logits, labels) # 计算损失
losses.append(loss.item())
correct = metric.compute(logits, labels) # 计算正确数
metric.update(correct) # 更新指标
acc = metric.accumulate() # 平均准确率
logger.info("eval loss: %.5f, acc: %.5f" % (np.mean(losses), acc))
metric.reset()
model.train()
return np.mean(losses),acc
def predict_sims(model, data_loader):
cosine_sims = []
model.eval()
with paddle.no_grad():
for batch_data in data_loader:
query_input_ids, query_token_type_ids, title_input_ids, title_token_type_ids = batch_data
batch_cosine_sim = model.cosine_sim( # [n]
query_input_ids=query_input_ids,
title_input_ids=title_input_ids,
query_token_type_ids=query_token_type_ids,
title_token_type_ids=title_token_type_ids).numpy()
cosine_sims.append(batch_cosine_sim)
cosine_sims = np.concatenate(cosine_sims, axis=0)
return cosine_sims
import os
import paddle
from paddlenlp.utils.log import logger
import pandas as pd
from tqdm import tqdm
from paddlenlp.datasets import load_dataset
from paddlenlp.prompt import InputExample
import json
def read_json_file(file):
with open(file, 'r', encoding='utf-8') as f:
for line in f.readlines():
line = line.strip()
过滤掉空内容的
if not line:
continue
line = json.loads(line)
yield line
def read_text_pair(data_path, is_test=False):
with open(data_path, 'r', encoding='utf-8') as f:
for line in f:
data = line.rstrip().split("\t")
if not is_test:
if len(data) != 3:
continue
query1,query2,label=data
if not query1.strip() or not query2.strip() or not label.strip():
continue
yield {'query1': query1, 'query2': query2, 'label': label}
else:
if not data[0].strip() or not data[1].strip():
continue
yield {'query1': data[0], 'query2': data[1]}
def read_pair_data(data_path):
with open(data_path, 'r', encoding='utf-8') as f:
for line in f.readlines():
word, label= line.strip().split('\t')
if not word.strip() or not str(label).strip():
continue
print(word,label)
yield {'text': word, 'label': label}
def get_label_dict(path):
with open(path, "r", encoding="utf-8") as f:
labels= [i.strip() for i in f.readlines()]
label2id=dict(zip(labels,range(len(labels))))
id2label=dict(zip(label2id.values(),label2id.keys()))
return label2id,id2label
def read_pair_by_pd(src_path):
df=pd.read_csv(src_path, sep='\t',header=None)
for index, row in df.iterrows():
query,title = row
yield {'query': str(query), 'title': str(title)}
对于一些不规则的三数据组,里面有引号,引号内可能有制表符的,用pd读
def read_data_by_pd(src_path, is_predict=False):
data = pd.read_csv(src_path, sep='\t')
for index, row in tqdm(data.iterrows()):
query = row['query']
title = row['title']
neg_title = row['neg_title']
yield {'query': query, 'title': title, 'neg_title': neg_title}
def read_data_by_pd_test(src_path, is_predict=False):
data = pd.read_csv(src_path, sep='\t')
for index, row in tqdm(data.iterrows()):
query = row['query']
title = row['title']
label = row['label']
yield {'query': query, 'title': title, 'label': label}
def read_texts(data_path, is_test=False):
with open(data_path, 'r', encoding='utf-8') as f:
for line in f:
data = line.rstrip().split("\t")
if not is_test:
if len(data) != 3:
continue
if len(data[0].strip()) == 0 or len(data[1].strip()) == 0 or len(data[2].strip()) == 0:
continue
yield {'text_a': data[0], 'text_b': data[1], 'label': data[2]}
else:
if len(data) != 2:
continue
if len(data[0].strip()) == 0 or len(data[1].strip()) == 0:
continue
yield {'text_a': data[0], 'text_b': data[1]}
def read_simcse_text(data_path):
with open(data_path,encoding='utf-8') as f:
for line in f:
data=line.strip().split('\t')
if len(data) != 2:
continue
query,title=data
if not query.strip() or not title.strip():
continue
yield {"query": query.strip(), "title": title.strip()}
def gen_text_file(similar_text_pair_file):
text2similar_text = {}
texts = []
with open(similar_text_pair_file, 'r', encoding='utf-8') as f:
for line in f:
splited_line = line.rstrip().split("\t")
if len(splited_line) != 2:
continue
text, similar_text = splited_line
if len(text.strip())==0 or len(similar_text.strip())==0:
continue
text2similar_text[text.strip()] = similar_text.strip()
texts.append({"text": text.strip()})
return texts, text2similar_text
def read_text_label(data_path):
with open(data_path,encoding='utf-8') as f:
for line in f:
split_line=line.rstrip().split('\t')
if len(split_line) !=2:
continue
if ' ' in split_line[0]:
text=split_line[0].replace(' ','_')
text=list(text)
label=split_line[1].split(' ')
assert len(text)==len(label), f'{text},{label}'
yield {"text": text, "label": label}
def load_dataset(datafiles):
def read(data_path):
with open(data_path, 'r', encoding='utf-8') as fp:
next(fp) # Skip header
for line in fp.readlines():
words, labels = line.strip('\n').split('\t')
words = words.split('\002')
labels = labels.split('\002')
yield words, labels
if isinstance(datafiles, str):
return MapDataset(list(read(datafiles)))
elif isinstance(datafiles, list) or isinstance(datafiles, tuple):
return [MapDataset(list(read(datafile))) for datafile in datafiles]
def gen_id2corpus(corpus_file):
id2corpus = {}
with open(corpus_file, 'r', encoding='utf-8') as f:
for idx, line in enumerate(f):
id2corpus[idx] = line.rstrip()
return id2corpus
def read_single_data(data_path):
with open(data_path, 'r', encoding='utf-8') as f:
for line in f.readlines():
word =line.rstrip()
if not word.strip():
continue
yield {'text': word}
def read_text_single(data_path):
with open(data_path, "r", encoding="utf-8") as f:
for line in f:
data = line.rstrip()
if not data:
continue
yield {"text_a": data, "text_b": data}