【python ROUGE BLEU jiaba.cut NLP常用的指标计算】

pip install -U nltk rouge jieba

# Rouge

from rouge import Rouge
rouge = Rouge()

reference = "The quick brown fox jumps over the lazy dog"
candidate = "The quick brown fox jumps over the dog"

scores = rouge.get_scores(candidate, reference)
scores

# jieba分词

import jieba


text = "我爱自然语言处理"
seg_list = list(jieba.cut(text, cut_all=False))

print("分词结果:", seg_list)

# Bleu

from nltk.translate.bleu_score import sentence_bleu

candidates = ["this", "is", "a", "test"]
references = [["this", "is", "a", "test"]]

bleu_score = sentence_bleu(references, candidates)
print(f"Corpus BLEU Score: {bleu_score}")