清洗文本高频词、情感分析、情感分类、主题建模挖掘主题

import pandas as pd

import re

import nltk

from nltk import FreqDist

from nltk.sentiment.vader import SentimentIntensityAnalyzer

from nltk.tokenize import word_tokenize

import spacy

from spacy.lang.en.stop_words import STOP_WORDS

from gensim.corpora import Dictionary

from gensim.models import LdaModel

下载NLTK的停用词、情感分析和词性标注所需的资源

nltk.download('stopwords')

nltk.download('punkt')

nltk.download('vader_lexicon')

加载SpaCy的英文NLP模型

nlp = spacy.load("en_core_web_sm")

读取Excel文件

df = pd.read_excel('nltk分词处理结果第二次.xlsx')

定义文本清洗函数

def clean_text(text):

去除HTML标签

cleaned_text = re.sub(r'<.*?>', '', text)

去除多余空格和换行符

cleaned_text = re.sub(r'\s+', ' ', cleaned_text)

转换为小写

cleaned_text = cleaned_text.lower()

return cleaned_text

清洗文本数据

df'cleaned_content' = df'content'.apply(clean_text)

词频分析

words = \[\]

for text in df'cleaned_content':

words += word_tokenize(text)

freq_dist = FreqDist(words)

print("词频分析结果：", freq_dist.most_common(10))

情感分析

sia = SentimentIntensityAnalyzer()

df'sentiment_score' = df'cleaned_content'.apply(lambda x: sia.polarity_scores(x)'compound')

print("情感分析结果：", df'sentiment_score')

定义阈值

positive_threshold = 0.5

negative_threshold = -0.5

根据情感分数进行分类

def classify_sentiment(score):

if score > positive_threshold:

return '积极'

elif score < negative_threshold:

return '消极'

else:

return '中性'

应用分类函数，创建新的列 'sentiment_category'

df'sentiment_category' = df'sentiment_score'.apply(classify_sentiment)

输出带有情感分类的数据

print(df\['cleaned_content', 'sentiment_score', 'sentiment_category'])

主题建模

tokens = \[token.text.lower() for token in nlp(text) if token.is_alpha and token.text.lower() not in STOP_WORDS for text in df'cleaned_content']

dictionary = Dictionary(tokens)

corpus = dictionary.doc2bow(text) for text in tokens

lda_model = LdaModel(corpus, num_topics=5, id2word=dictionary, passes=15)

topics = lda_model.print_topics(num_words=5)

print("主题建模结果：")

for topic in topics:

print(topic)