清洗文本高频词、情感分析、情感分类、主题建模挖掘主题

import pandas as pd

import re

import nltk

from nltk import FreqDist

from nltk.sentiment.vader import SentimentIntensityAnalyzer

from nltk.tokenize import word_tokenize

import spacy

from spacy.lang.en.stop_words import STOP_WORDS

from gensim.corpora import Dictionary

from gensim.models import LdaModel

下载NLTK的停用词、情感分析和词性标注所需的资源

nltk.download('stopwords')

nltk.download('punkt')

nltk.download('vader_lexicon')

加载SpaCy的英文NLP模型

nlp = spacy.load("en_core_web_sm")

读取Excel文件

df = pd.read_excel('nltk分词处理结果第二次.xlsx')

定义文本清洗函数

def clean_text(text):

去除HTML标签

cleaned_text = re.sub(r'<.*?>', '', text)

去除多余空格和换行符

cleaned_text = re.sub(r'\s+', ' ', cleaned_text)

转换为小写

cleaned_text = cleaned_text.lower()

return cleaned_text

清洗文本数据

df['cleaned_content'] = df['content'].apply(clean_text)

词频分析

words = []

for text in df['cleaned_content']:

words += word_tokenize(text)

freq_dist = FreqDist(words)

print("词频分析结果：", freq_dist.most_common(10))

情感分析

sia = SentimentIntensityAnalyzer()

df['sentiment_score'] = df['cleaned_content'].apply(lambda x: sia.polarity_scores(x)['compound'])

print("情感分析结果：", df['sentiment_score'])

定义阈值

positive_threshold = 0.5

negative_threshold = -0.5

根据情感分数进行分类

def classify_sentiment(score):

if score > positive_threshold:

return '积极'

elif score < negative_threshold:

return '消极'

else:

return '中性'

应用分类函数，创建新的列 'sentiment_category'

df['sentiment_category'] = df['sentiment_score'].apply(classify_sentiment)

输出带有情感分类的数据

print(df[['cleaned_content', 'sentiment_score', 'sentiment_category']])

主题建模

tokens = [[token.text.lower() for token in nlp(text) if token.is_alpha and token.text.lower() not in STOP_WORDS] for text in df['cleaned_content']]

dictionary = Dictionary(tokens)

corpus = [dictionary.doc2bow(text) for text in tokens]

lda_model = LdaModel(corpus, num_topics=5, id2word=dictionary, passes=15)

topics = lda_model.print_topics(num_words=5)

print("主题建模结果：")

for topic in topics:

print(topic)