from time import sleep
from selenium.webdriver.chrome.service import Service
from selenium.webdriver import Chrome,ChromeOptions
from selenium.webdriver.common.by import By
import warnings
import csv
import random
#忽略警告
warnings.filterwarnings("ignore")def main():driver.get('https://travel.qunar.com/p-oi7482144-datangbuyecheng')sleep(10)for i in range(100):move_page(driver)item_list = driver.find_elements(By.XPATH,'//*[@id="comment_box"]/li')for item in item_list:try:comment_time = item.find_element(By.XPATH,'./div[1]/div[1]/div[5]/ul/li[1]').textcomment_text = item.find_element(By.XPATH,'./div[1]/div[1]/div[1]/a').text +' ' + ''.join(item.find_element(By.XPATH,'./div[1]/div[1]/div[3]/p').text)comment = comment_text.replace('\n','')csvwriter.writerow((comment_time,comment))f.flush()print(comment_time)except Exception as e:comment_time = item.find_element(By.XPATH,'./div[1]/div[1]/div[4]/ul/li[1]').textcomment_text = item.find_element(By.XPATH,'./div[1]/div[1]/div[1]/a').text +' ' + ''.join(item.find_element(By.XPATH,'./div[1]/div[1]/div[3]/p').text)comment = comment_text.replace('\n','')csvwriter.writerow((comment_time,comment))f.flush()print(comment_time)passdriver.find_element(By.LINK_TEXT,'下一页').click()print(f'=====================第{i+1}页爬取完毕！=========================')sleep(random.random()*10)def move_page(web):"""执行页面滚动的操作"""  # javascriptfor x in range(1, 9, 2):  # 1 3 5 7 9 sleep(0.5)j = x / 9  # 1/9  3/9  5/9  9/9js = 'document.documentElement.scrollTop = document.documentElement.scrollHeight * %f' % jweb.execute_script(js)

import collections
import re
import jieba# 统计词频
def Statistical_word_frequency(df,filename):data =  ''.join([re.sub(r'[^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z]：','',item) for item in df])# 文本预处理 ：去除一些无用的字符只提取出中文出来new_data = re.findall('[\u4e00-\u9fa5]+', data, re.S)new_data = "".join(new_data)# 文本分词seg_list_exact = jieba.cut(new_data)result_list = []with open('停用词库.txt', encoding='utf-8') as f: #可根据需要打开停用词库，然后加上不想显示的词语con = f.readlines()stop_words = set()for i in con:i = i.replace("\n", "")   # 去掉读取每一行数据的\nstop_words.add(i)for word in seg_list_exact:if word not in stop_words and len(word) > 1:result_list.append(word)word_counts = collections.Counter(result_list)# 词频统计：获取前20最高频的词word_counts_top = word_counts.most_common(20)with open(f'{filename}.txt','w',encoding='utf-8')as f:for i in word_counts_top:f.write(str(i[0]))f.write('\t')f.write(str(i[1]))f.write('\n')

import collections
import stylecloud
import re
import jieba
from PIL import Imagedef draw_WorldCloud(df,pic_name,color='white'):data =  ''.join([re.sub(r'[^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z]：','',item) for item in df])# 文本预处理 ：去除一些无用的字符只提取出中文出来new_data = re.findall('[\u4e00-\u9fa5]+', data, re.S)new_data = "".join(new_data)# 文本分词seg_list_exact = jieba.cut(new_data)result_list = []with open('停用词库.txt', encoding='utf-8') as f: #可根据需要打开停用词库，然后加上不想显示的词语con = f.readlines()stop_words = set()for i in con:i = i.replace("\n", "")   # 去掉读取每一行数据的\nstop_words.add(i)for word in seg_list_exact:if word not in stop_words and len(word) > 1:result_list.append(word)# 绘制词云图stylecloud.gen_stylecloud(text=' '.join(result_list), collocations=False, # 是否包括两个单词的搭配（二字组）font_path=r'C:\Windows\Fonts\msyh.ttc', #设置字体，参考位置为  size=800, # stylecloud 的大小palette='cartocolors.qualitative.Bold_7', # 调色板background_color=color, # 背景颜色icon_name='fas fa-cloud', # 形状的图标名称 gradient='horizontal', # 梯度方向max_words=2000, # stylecloud 可包含的最大单词数max_font_size=150, # stylecloud 中的最大字号stopwords=True, # 布尔值，用于筛除常见禁用词output_name=f'{pic_name}.png') # 输出图片# 打开图片展示img=Image.open(f'{pic_name}.png')img.show()draw_WorldCloud(data[data['label']==0]['评论内容'],'主题0词云图')
draw_WorldCloud(data[data['label']==1]['评论内容'],'主题1词云图')
draw_WorldCloud(data[data['label']==2]['评论内容'],'主题2词云图')

from pyecharts.charts import Bar,Line
from pyecharts import options as optsx = ['Topic-0', 'Topic-1', 'Topic-2']
bar = Bar()
bar.add_xaxis(x)
bar.add_yaxis('发文量',
[714, 673, 483],
label_opts=opts.LabelOpts(is_show=False))
bar.set_global_opts(tooltip_opts=opts.TooltipOpts(is_show=True,trigger='axis',axis_pointer_type='cross'),xaxis_opts= opts.AxisOpts(type_='category',axispointer_opts=opts.AxisPointerOpts(is_show=True,type_='shadow')))bar.extend_axis(yaxis=opts.AxisOpts(name='主题强度',min_=0,max_= 0.4,interval = 5,axislabel_opts = opts.LabelOpts()
))line = Line()
line.add_xaxis(x)
line.add_yaxis('主题强度',
[0.38181818181818183, 0.3598930481283422, 0.25828877005347595],
yaxis_index=1,
label_opts=opts.LabelOpts(is_show=False)
)bar.overlap(line) # 合并图
bar.render_notebook()

合并数据集
import pandas as pddf1 = pd.read_csv('./去哪儿网爬虫/comments.csv')
df1.head()
df2 = pd.read_csv('./携程爬虫/comments.csv',usecols=[0,2])
df2.head()
new_df = pd.concat([df1,df2],axis=0)
new_df.to_csv('dtbyc_comments.csv',index=False)import pandas as pd
import matplotlib.pylab as plt
import numpy as np
import seaborn as sns
sns.set(font='SimHei')
plt.rcParams['font.sans-serif'] = ['SimHei'] #解决中文显示
plt.rcParams['axes.unicode_minus'] = False   #解决符号无法显示
import warnings
warnings.filterwarnings('ignore')data = pd.read_csv('dtbyc_comments.csv')
data.head()
data.shape
data.isnull().sum() # 统计缺失值
data.duplicated().sum() # 统计重复值
data.dropna(inplace=True) # 删除缺失值
data.isnull().sum() # 统计缺失值
data.drop_duplicates(inplace=True) # 删除重复值
data.duplicated().sum()
# 处理评论时间
data['评论时间'] = data['评论时间'].apply(lambda x:x.split('发表于')[0])
# 加载情感分析模块
from snownlp import SnowNLP
# 遍历每条评论进行预测
values=[SnowNLP(i).sentiments for i in data['评论内容']]
# 输出积极的概率
# myval保存预测值
myval=[]
good=0
mid=0
bad=0
for i in values:if (i>=0.7):myval.append("积极")good=good+1elif 0.3<i<0.7:myval.append("中性")mid+=1else:myval.append("消极")bad=bad+1
data['预测值']=values
data['评价类别']=myval
data.head()
rate=good/(good+bad+mid)
print('好评率','%.f%%' % (rate * 100)) #格式化为百分比
#作图
y=values
plt.rc('font', family='SimHei', size=10)
plt.plot(y, marker='o', mec='r', mfc='w',label=u'评价分值')
plt.xlabel('用户')
plt.ylabel('评价分值')
# 让图例生效
plt.legend()
#添加标题
plt.title('评论情感分析-redmi',family='SimHei',size=14,color='blue')
plt.show()
# 情感分类占比
y = data['评价类别'].value_counts().values.tolist()
plt.pie(y,labels=['积极','中性','消极'], # 设置饼图标签colors=["#d5695d", "#5d8ca8", "#65a479"], # 设置饼图颜色autopct='%.2f%%', # 格式化输出百分比)
plt.title('评论情感分析',family='SimHei',size=14,color='blue')
plt.show()
import collections
import re
import jieba# 统计词频
def Statistical_word_frequency(df,filename):data =  ''.join([re.sub(r'[^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z]：','',item) for item in df])# 文本预处理 ：去除一些无用的字符只提取出中文出来new_data = re.findall('[\u4e00-\u9fa5]+', data, re.S)new_data = "".join(new_data)# 文本分词seg_list_exact = jieba.cut(new_data)result_list = []with open('停用词库.txt', encoding='utf-8') as f: #可根据需要打开停用词库，然后加上不想显示的词语con = f.readlines()stop_words = set()for i in con:i = i.replace("\n", "")   # 去掉读取每一行数据的\nstop_words.add(i)for word in seg_list_exact:if word not in stop_words and len(word) > 1:result_list.append(word)word_counts = collections.Counter(result_list)# 词频统计：获取前20最高频的词word_counts_top = word_counts.most_common(20)with open(f'{filename}.txt','w',encoding='utf-8')as f:for i in word_counts_top:f.write(str(i[0]))f.write('\t')f.write(str(i[1]))f.write('\n')
Statistical_word_frequency(df=data[data['评价类别']=='积极']['评论内容'],filename='积极情绪词频统计')
Statistical_word_frequency(df=data[data['评价类别']=='中性']['评论内容'],filename='中性情绪词频统计')
Statistical_word_frequency(df=data[data['评价类别']=='消极']['评论内容'],filename='消极情绪词频统计')
# 提取出特定的情绪文本内容并另存为txt文件
def extract_text(df,filename):with open(f'{filename}.txt','w',encoding='utf-8')as f:for i in df:f.write(i)f.write('\n')
extract_text(data[data['评价类别']=='积极']['评论内容'],filename='积极情绪文本')
extract_text(data[data['评价类别']=='中性']['评论内容'],filename='中性情绪文本')
extract_text(data[data['评价类别']=='消极']['评论内容'],filename='消极情绪文本')LDA主题分析
import re
import jiebadef chinese_word_cut(mytext):jieba.load_userdict('dic.txt')  # 这里你可以添加jieba库识别不了的网络新词，避免将一些新词拆开jieba.initialize()# 文本预处理 ：去除一些无用的字符只提取出中文出来new_data = re.findall('[\u4e00-\u9fa5]+', mytext, re.S)new_data = " ".join(new_data)# 文本分词seg_list_exact = jieba.cut(new_data)result_list = []with open('停用词库.txt', encoding='utf-8') as f: # 可根据需要打开停用词库，然后加上不想显示的词语con = f.readlines()stop_words = set()for i in con:i = i.replace("\n", "")   # 去掉读取每一行数据的\nstop_words.add(i)for word in seg_list_exact:if word not in stop_words and len(word) > 1:result_list.append(word)      return " ".join(result_list)test = '在这里选购安全方便快捷💜，快递小哥态度非常好，这个价格非常给力💪，十分推荐，会继续回购😃大品牌！值得信赖！效果超级好！贵有贵的道理！很棒的商品！'
chinese_word_cut(test)
# 中文分词
data["content_cutted"] = data['评论内容'].apply(chinese_word_cut)
data.head()
import tomotopy as tpdef find_k(docs, min_k=1, max_k=20, min_df=2):# min_df 词语最少出现在2个文档中scores = []for k in range(min_k, max_k):mdl = tp.LDAModel(min_df=min_df, k=k, seed=555)for words in docs:if words:mdl.add_doc(words)mdl.train(20)coh = tp.coherence.Coherence(mdl)scores.append(coh.get_score())plt.plot(range(min_k, max_k), scores)plt.xlabel("number of topics")plt.ylabel("coherence")plt.show()find_k(docs=data['content_cutted'], min_k=1, max_k=10, min_df=2)
import tomotopy as tp
# 初始化LDA
mdl = tp.LDAModel(k=3, min_df=2, seed=555)
for words in data['content_cutted']:#确认words 是 非空词语列表if words:mdl.add_doc(words=words.split())#训 练
mdl.train()
# 查看每个topic feature words
wordcloud_words = []
for k in range(mdl.k):with open(f'主题{k}特征词及其权重.txt','w',encoding='utf-8')as f:print('Top 20 words of topic #{}'.format(k))print(mdl.get_topic_words(k, top_n=25))for item in mdl.get_topic_words(k, top_n=50): # 将主题的前50个特征词及其权重进行保存f.write(str(item[0]))f.write('\t')f.write(str(item[1]))f.write('\n')# 将每个主题的关键词及其权重(放大10000倍)进行保存，后面可以做出词云图topic_words_list = [(item[0],int(item[1]*10000)) for item in mdl.get_topic_words(k, top_n=500)]wordcloud_words.append(topic_words_list)
# 查看话题模型信息
mdl.summary()
import pyLDAvis
import numpy as np# 在notebook显示
pyLDAvis.enable_notebook()# 获取pyldavis需要的参数
topic_term_dists = np.stack([mdl.get_topic_word_dist(k) for k in range(mdl.k)])
doc_topic_dists = np.stack([doc.get_topic_dist() for doc in mdl.docs])
doc_topic_dists /= doc_topic_dists.sum(axis=1, keepdims=True)
doc_lengths = np.array([len(doc.words) for doc in mdl.docs])
vocab = list(mdl.used_vocabs)
term_frequency = mdl.used_vocab_freqprepared_data = pyLDAvis.prepare(topic_term_dists, doc_topic_dists, doc_lengths, vocab, term_frequency,start_index=0, # tomotopy话题id从0开始，pyLDAvis话题id从1开始sort_topics=False # 注意：否则pyLDAvis与tomotopy内的话题无法一一对应。 
)# 可视化结果存到html文件中
pyLDAvis.save_html(prepared_data, 'ldavis.html')# notebook中显示
pyLDAvis.display(prepared_data)
主题强度分析
from gensim import corpora
data_set=[]  #建立存储分词的列表
for i in range(len(data['content_cutted'].to_list())):result=[]seg_list = data['content_cutted'].to_list()[i].split()for w in seg_list :  #读取每一行分词result.append(w)data_set.append(result)
# print(data_set)
dictionary = corpora.Dictionary(data_set)  # 构建词典
corpus = [dictionary.doc2bow(text) for text in data_set]  #表示为第几个单词出现了几次
from gensim.models.ldamodel import LdaModel
lda = LdaModel(corpus=corpus, id2word=dictionary, num_topics=3, passes = 30,random_state=1)
label_list = [] 
for i in lda.get_document_topics(corpus)[:]:listj=[]for j in i:listj.append(j[1])bz=listj.index(max(listj))label_list.append(i[bz][0])
data['label'] = label_list
data.head()
from gensim.models.ldamodel import LdaModel
lda = LdaModel(corpus=corpus, id2word=dictionary, num_topics=3, passes = 30,random_state=1)
label_list = [] 
for i in lda.get_document_topics(corpus)[:]:listj=[]for j in i:listj.append(j[1])bz=listj.index(max(listj))label_list.append(i[bz][0])
data['label'] = label_list
data.head()
import collections
import stylecloud
import re
import jieba
from PIL import Imagedef draw_WorldCloud(df,pic_name,color='white'):data =  ''.join([re.sub(r'[^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z][^0-9A-Za-z]：','',item) for item in df])# 文本预处理 ：去除一些无用的字符只提取出中文出来new_data = re.findall('[\u4e00-\u9fa5]+', data, re.S)new_data = "".join(new_data)# 文本分词seg_list_exact = jieba.cut(new_data)result_list = []with open('停用词库.txt', encoding='utf-8') as f: #可根据需要打开停用词库，然后加上不想显示的词语con = f.readlines()stop_words = set()for i in con:i = i.replace("\n", "")   # 去掉读取每一行数据的\nstop_words.add(i)for word in seg_list_exact:if word not in stop_words and len(word) > 1:result_list.append(word)# 绘制词云图stylecloud.gen_stylecloud(text=' '.join(result_list), collocations=False, # 是否包括两个单词的搭配（二字组）font_path=r'C:\Windows\Fonts\msyh.ttc', #设置字体，参考位置为  size=800, # stylecloud 的大小palette='cartocolors.qualitative.Bold_7', # 调色板background_color=color, # 背景颜色icon_name='fas fa-cloud', # 形状的图标名称 gradient='horizontal', # 梯度方向max_words=2000, # stylecloud 可包含的最大单词数max_font_size=150, # stylecloud 中的最大字号stopwords=True, # 布尔值，用于筛除常见禁用词output_name=f'{pic_name}.png') # 输出图片# 打开图片展示img=Image.open(f'{pic_name}.png')img.show()
draw_WorldCloud(data[data['label']==0]['评论内容'],'主题0词云图')
draw_WorldCloud(data[data['label']==1]['评论内容'],'主题1词云图')
draw_WorldCloud(data[data['label']==2]['评论内容'],'主题2词云图')
data['label'].value_counts()
data['label'].value_counts()/1870
from pyecharts.charts import Bar,Line
from pyecharts import options as optsx = ['Topic-0', 'Topic-1', 'Topic-2']
bar = Bar()
bar.add_xaxis(x)
bar.add_yaxis('发文量',
[714, 673, 483],
label_opts=opts.LabelOpts(is_show=False))
bar.set_global_opts(tooltip_opts=opts.TooltipOpts(is_show=True,trigger='axis',axis_pointer_type='cross'),xaxis_opts= opts.AxisOpts(type_='category',axispointer_opts=opts.AxisPointerOpts(is_show=True,type_='shadow')))bar.extend_axis(yaxis=opts.AxisOpts(name='主题强度',min_=0,max_= 0.4,interval = 5,axislabel_opts = opts.LabelOpts()
))line = Line()
line.add_xaxis(x)
line.add_yaxis('主题强度',
[0.38181818181818183, 0.3598930481283422, 0.25828877005347595],
yaxis_index=1,
label_opts=opts.LabelOpts(is_show=False)
)bar.overlap(line) # 合并图
bar.render_notebook()
data['评论年份'] = data['评论时间'].apply(lambda x:x.split('-')[0]) # 提取年份
# 准备数据
df_0 = data[data['label']==0].groupby(by='评论年份')['评论内容'].count()
print(df_0)
plt.figure(figsize=(12,6))
y = df_0.values.tolist()
x = range(12)
x_ticks = [f'{i}年' for i in df_0.index.to_list()]
plt.xticks(x,x_ticks)
#添加标题
plt.title('各年份的发文量')
plt.xlabel('年份')
plt.ylabel('发文量')
plt.plot(x,y,label='Topic-0')
for a,b in zip(df_0.reset_index(drop=True).index,df_0.reset_index(drop=True).values):plt.text(a,b,'%d'%b,ha='center',va='bottom',fontsize=14)
plt.legend()
plt.show()
# 准备数据
df_1 = data[data['label']==1].groupby(by='评论年份')['评论内容'].count()
print(df_1)
plt.figure(figsize=(12,6))
y = df_1.values.tolist()
x = range(12)
x_ticks = [f'{i}年' for i in df_1.index.to_list()]
plt.xticks(x,x_ticks)
#添加标题
plt.title('各年份的发文量')
plt.xlabel('年份')
plt.ylabel('发文量')
plt.plot(x,y,label='Topic-1')
for a,b in zip(df_1.reset_index(drop=True).index,df_1.reset_index(drop=True).values):plt.text(a,b,'%d'%b,ha='center',va='bottom',fontsize=14)
plt.legend()
plt.show()
# 准备数据
df_2 = data[data['label']==1].groupby(by='评论年份')['评论内容'].count()
print(df_2)
plt.figure(figsize=(12,6))
y = df_2.values.tolist()
x = range(12)
x_ticks = [f'{i}年' for i in df_2.index.to_list()]
plt.xticks(x,x_ticks)
#添加标题
plt.title('各年份的发文量')
plt.xlabel('年份')
plt.ylabel('发文量')
plt.plot(x,y,label='Topic-2')
for a,b in zip(df_2.reset_index(drop=True).index,df_2.reset_index(drop=True).values):plt.text(a,b,'%d'%b,ha='center',va='bottom',fontsize=14)
plt.legend()
plt.show()