Duee1.0信息提取句子级数据预处理

import os

import sys

import json

def read_by_lines(path):

result = list()

with open(path, "r", encoding="utf8") as infile:

for line in infile:

result.append(line.strip())

return result

def write_by_lines(path, data):

with open(path, "w", encoding="utf8") as outfile:

outfile.write(d + "\\n") for d in data

def data_process(path, model="trigger", is_predict=False):

def label_data(data, start, l, _type):

for i in range(start, start + l):#从起始索引到结束

suffix = "B-" if i == start else "I-" #前缀

data[i] = "{}{}".format(suffix, _type)

return data

sentences = []

output = ["text_a"] if is_predict else ["text_a\tlabel"]#文本,标签

with open(path) as f:

for line in f:

d_json = json.loads(line.strip())#每一行

_id = d_json["id"]#id

text_a = [

"," if t == " " or t == "\n" or t == "\t" else t

for t in list(d_json["text"].lower())

]# 文本

if is_predict:

sentences.append({"text": d_json["text"], "id": _id})

output.append('\002'.join(text_a))

else:

if model == "trigger":

labels = ["O"] * len(text_a)#标签初始化为全部非实体

if len(d_json.get("event_list", [])) == 0:

continue

for event in d_json.get("event_list"):

event_type = event["event_type"]#事件类型

start = event["trigger_start_index"]#触发词起始索引

trigger = event["trigger"]#触发池

#为触发词设置labels

labels = label_data(labels, start, len(trigger),

event_type)

output.append("{}\t{}".format('\002'.join(text_a),

'\002'.join(labels)))

elif model == "role":

labels = ["O"] * len(text_a)#标签

if len(d_json.get("event_list", [])) == 0:

continue

for event in d_json.get("event_list"):

for arg in event["arguments"]:

role_type = arg["role"]#论元角色类型

argument = arg["argument"]#论元

start = arg["argument_start_index"]#论元起始

labels = label_data(labels, start, len(argument),

role_type)

output.append("{}\t{}".format('\002'.join(text_a),

'\002'.join(labels)))

return output

def schema_process(path, model="trigger"):

def label_add(labels, _type):

if "B-{}".format(_type) not in labels:#不在里面就添加

labels.extend(["B-{}".format(_type), "I-{}".format(_type)])#B-,I-

return labels

labels = []#存放事件类型标签或角色标签

for line in read_by_lines(path):

d_json = json.loads(line.strip())

if model == "trigger":

labels = label_add(labels, d_json["event_type"])

elif model == "role":

for role in d_json["role_list"]:

labels = label_add(labels, role["role"])

labels.append("O")

tags = []

for index, label in enumerate(labels):

tags.append("{}\t{}".format(index, label))

return tags

conf_dir = "./conf/DuEE1.0"

schema_path ='./datasets/DuEE_1_0/event_schema.json'

tags_trigger_path = "{}/trigger_tag.dict".format(conf_dir)

tags_role_path = "{}/role_tag.dict".format(conf_dir)

read_by_lines(schema_path)[0]

!unzip DuEE_1_0.zip -d ./datasets/

tags_trigger = schema_process(schema_path, "trigger")

os.makedirs(conf_dir,exist_ok=True)

write_by_lines(tags_trigger_path, tags_trigger)

tags_role = schema_process(schema_path, "role")

write_by_lines(tags_role_path, tags_role)

data_dir = "./datasets/DuEE1.0"

trigger_save_dir = "{}/trigger".format(data_dir)

role_save_dir = "{}/role".format(data_dir)

if not os.path.exists(trigger_save_dir):

os.makedirs(trigger_save_dir)

if not os.path.exists(role_save_dir):

os.makedirs(role_save_dir)

train_tri = data_process("./datasets/DuEE_1_0/train.json","trigger")

read_by_lines('./datasets/DuEE_1_0/train.json')[0]

write_by_lines("{}/train.tsv".format(trigger_save_dir), train_tri)

dev_tri = data_process("./datasets/DuEE_1_0/dev.json","trigger")

write_by_lines("{}/dev.tsv".format(trigger_save_dir), dev_tri)

test_tri = data_process("./datasets/DuEE_1_0/test.json", "trigger")

write_by_lines("{}/test.tsv".format(trigger_save_dir), test_tri)

train_role = data_process("./datasets/DuEE_1_0/train.json", "role")

write_by_lines("{}/train.tsv".format(role_save_dir), train_role)

dev_role = data_process("./datasets/DuEE_1_0/dev.json", "role")

write_by_lines("{}/dev.tsv".format(role_save_dir), dev_role)

test_role = data_process("./datasets/DuEE_1_0/test.json", "role")

write_by_lines("{}/test.tsv".format(role_save_dir), test_role)

相关推荐
DeepSeek大模型官方教程4 小时前
NLP之文本纠错开源大模型:兼看语音大模型总结
大数据·人工智能·ai·自然语言处理·大模型·产品经理·大模型学习
onceco15 小时前
领域LLM九讲——第5讲 为什么选择OpenManus而不是QwenAgent(附LLM免费api邀请码)
人工智能·python·深度学习·语言模型·自然语言处理·自动化
陈敬雷-充电了么-CEO兼CTO1 天前
大模型技术原理 - 基于Transformer的预训练语言模型
人工智能·深度学习·语言模型·自然语言处理·chatgpt·aigc·transformer
AI让世界更懂你1 天前
【ACL系列论文写作指北15-如何进行reveiw】-公平、公正、公开
人工智能·自然语言处理
**梯度已爆炸**1 天前
自然语言处理入门
人工智能·自然语言处理
羊小猪~~1 天前
【NLP入门系列五】中文文本分类案例
人工智能·深度学习·考研·机器学习·自然语言处理·分类·数据挖掘
xwz小王子1 天前
从LLM到WM:大语言模型如何进化成具身世界模型?
人工智能·语言模型·自然语言处理
静心问道1 天前
FLAN-T5:规模化指令微调的语言模型
人工智能·语言模型·自然语言处理
李师兄说大模型1 天前
KDD 2025 | 地理定位中的群体智能:一个多智能体大型视觉语言模型协同框架
人工智能·深度学习·机器学习·语言模型·自然语言处理·大模型·deepseek
静心问道1 天前
SqueezeBERT:计算机视觉能为自然语言处理在高效神经网络方面带来哪些启示?
人工智能·计算机视觉·自然语言处理