Duee1.0信息提取句子级数据预处理

import os

import sys

import json

def read_by_lines(path):

result = list()

with open(path, "r", encoding="utf8") as infile:

for line in infile:

result.append(line.strip())

return result

def write_by_lines(path, data):

with open(path, "w", encoding="utf8") as outfile:

outfile.write(d + "\\n") for d in data

def data_process(path, model="trigger", is_predict=False):

def label_data(data, start, l, _type):

for i in range(start, start + l):#从起始索引到结束

suffix = "B-" if i == start else "I-" #前缀

data[i] = "{}{}".format(suffix, _type)

return data

sentences = []

output = ["text_a"] if is_predict else ["text_a\tlabel"]#文本,标签

with open(path) as f:

for line in f:

d_json = json.loads(line.strip())#每一行

_id = d_json["id"]#id

text_a = [

"," if t == " " or t == "\n" or t == "\t" else t

for t in list(d_json["text"].lower())

]# 文本

if is_predict:

sentences.append({"text": d_json["text"], "id": _id})

output.append('\002'.join(text_a))

else:

if model == "trigger":

labels = ["O"] * len(text_a)#标签初始化为全部非实体

if len(d_json.get("event_list", [])) == 0:

continue

for event in d_json.get("event_list"):

event_type = event["event_type"]#事件类型

start = event["trigger_start_index"]#触发词起始索引

trigger = event["trigger"]#触发池

#为触发词设置labels

labels = label_data(labels, start, len(trigger),

event_type)

output.append("{}\t{}".format('\002'.join(text_a),

'\002'.join(labels)))

elif model == "role":

labels = ["O"] * len(text_a)#标签

if len(d_json.get("event_list", [])) == 0:

continue

for event in d_json.get("event_list"):

for arg in event["arguments"]:

role_type = arg["role"]#论元角色类型

argument = arg["argument"]#论元

start = arg["argument_start_index"]#论元起始

labels = label_data(labels, start, len(argument),

role_type)

output.append("{}\t{}".format('\002'.join(text_a),

'\002'.join(labels)))

return output

def schema_process(path, model="trigger"):

def label_add(labels, _type):

if "B-{}".format(_type) not in labels:#不在里面就添加

labels.extend(["B-{}".format(_type), "I-{}".format(_type)])#B-,I-

return labels

labels = []#存放事件类型标签或角色标签

for line in read_by_lines(path):

d_json = json.loads(line.strip())

if model == "trigger":

labels = label_add(labels, d_json["event_type"])

elif model == "role":

for role in d_json["role_list"]:

labels = label_add(labels, role["role"])

labels.append("O")

tags = []

for index, label in enumerate(labels):

tags.append("{}\t{}".format(index, label))

return tags

conf_dir = "./conf/DuEE1.0"

schema_path ='./datasets/DuEE_1_0/event_schema.json'

tags_trigger_path = "{}/trigger_tag.dict".format(conf_dir)

tags_role_path = "{}/role_tag.dict".format(conf_dir)

read_by_lines(schema_path)[0]

!unzip DuEE_1_0.zip -d ./datasets/

tags_trigger = schema_process(schema_path, "trigger")

os.makedirs(conf_dir,exist_ok=True)

write_by_lines(tags_trigger_path, tags_trigger)

tags_role = schema_process(schema_path, "role")

write_by_lines(tags_role_path, tags_role)

data_dir = "./datasets/DuEE1.0"

trigger_save_dir = "{}/trigger".format(data_dir)

role_save_dir = "{}/role".format(data_dir)

if not os.path.exists(trigger_save_dir):

os.makedirs(trigger_save_dir)

if not os.path.exists(role_save_dir):

os.makedirs(role_save_dir)

train_tri = data_process("./datasets/DuEE_1_0/train.json","trigger")

read_by_lines('./datasets/DuEE_1_0/train.json')[0]

write_by_lines("{}/train.tsv".format(trigger_save_dir), train_tri)

dev_tri = data_process("./datasets/DuEE_1_0/dev.json","trigger")

write_by_lines("{}/dev.tsv".format(trigger_save_dir), dev_tri)

test_tri = data_process("./datasets/DuEE_1_0/test.json", "trigger")

write_by_lines("{}/test.tsv".format(trigger_save_dir), test_tri)

train_role = data_process("./datasets/DuEE_1_0/train.json", "role")

write_by_lines("{}/train.tsv".format(role_save_dir), train_role)

dev_role = data_process("./datasets/DuEE_1_0/dev.json", "role")

write_by_lines("{}/dev.tsv".format(role_save_dir), dev_role)

test_role = data_process("./datasets/DuEE_1_0/test.json", "role")

write_by_lines("{}/test.tsv".format(role_save_dir), test_role)

相关推荐
java1234_小锋1 小时前
【NLP舆情分析】基于python微博舆情分析可视化系统(flask+pandas+echarts) 视频教程 - 微博评论IP地图可视化分析实现
python·自然语言处理·flask
java1234_小锋5 小时前
【NLP舆情分析】基于python微博舆情分析可视化系统(flask+pandas+echarts) 视频教程 - 微博文章数据可视化分析-文章评论量分析实现
python·自然语言处理·flask
java1234_小锋7 小时前
【NLP舆情分析】基于python微博舆情分析可视化系统(flask+pandas+echarts) 视频教程 - 微博内容IP地图可视化分析实现
python·自然语言处理·flask
AI扶我青云志7 小时前
BERT 的 NSP慢慢转换为SOP
人工智能·自然语言处理·llm
真实的菜8 小时前
大语言模型(LLM):重塑AI未来的技术革命
人工智能·语言模型·自然语言处理
sonrisa_13 小时前
大语言模型API付费?
人工智能·语言模型·自然语言处理
Allen_LVyingbo14 小时前
智能AI医疗物资/耗材管理系统升级改造方案分析
人工智能·自然语言处理·开源·健康医疗·扩展屏应用开发
wa的一声哭了14 小时前
Python多进程并行multiprocess基础
开发语言·jvm·人工智能·python·机器学习·语言模型·自然语言处理
cwn_1 天前
自然语言处理NLP (1)
人工智能·深度学习·机器学习·自然语言处理
金井PRATHAMA1 天前
广泛分布于内侧内嗅皮层全层的速度细胞(speed cells)对NLP中的深层语义分析的积极影响和启示
人工智能·神经网络·自然语言处理·知识图谱