python爬取京东商品信息与可视化

项目介绍：使用python爬取京东电商拿到价格、店铺、链接、销量并做可视化

...........................................................................................................................................................

|------|
| 项目介绍 |
| 效果展示 |
| 全部代码 |

...........................................................................................................................................................

效果展示：

...........................................................................................................................................................

..........................................................................................................................................................

价格与店铺可视化：

...........................................................................................................................................................

..........................................................................................................................................................

销量与店铺可视化：

..........................................................................................................................................................

..........................................................................................................................................................

爬取主函数：

..........................................................................................................................................................

import selenium.webdriver as driver
from selenium.webdriver.common.by import By
import time
from lxml import etree
import pandas

class GetData:
    """一手数据获取：前端代码"""
    def __init__(self):
        # 目标网站：京东iPhone4s搜索页面
        self.url = 'https://search.jd.com/Search?keyword=%E5%94%90%E5%8D%A1%E5%90%8A%E5%9D%A0&enc=utf-8&wq=%E5%94%90%E5%8D%A1%E5%90%8A%E5%9D%A0&pvid=31f3e974663949f39b95db6bb05ad3f8'
        # 创建浏览器
        self.edge = driver.Edge()
        # 访问指定页面
        self.edge.get(self.url)

    def take(self):
        '对页面进行操作'
        button = self.edge.find_element(By.CLASS_NAME,'weixin-icon')
        button.click()
        # 等待登录
        time.sleep(10)
        # 最终数据:目标页面代码
        self.over_data = self.edge.page_source

class Sift:
    "筛选信息"
    def __init__(self):
        # 创建GetData类获取前端代码
        geter = GetData()   # 创建
        geter.take()    # 操作
        # 最终的前端页面数据
        self.over_data = geter.over_data

    def take(self):
        # 创建xpath解析器
        html = etree.HTML(self.over_data)
        # 获取数据
        self.prices = html.xpath('//*[@id="J_goodsList"]/ul/li[*]/div/div[*]/strong/i/text()')
        self.shop = html.xpath('//*[@id="J_goodsList"]/ul/li[*]/div/div[*]/span/a/text()')
        self.shopping = html.xpath('//*[@id="J_goodsList"]/ul/li[*]/div/div[*]/span/a/@href')
        self.ping = html.xpath('/html/body/div[*]/div[*]/div[*]/div[*]/div/div[*]/ul/li[*]/div/div[*]/strong/a/text()')
        # 将网站链接手动加上https:
        for i in range(len(self.shopping)):
            data = 'https:'+self.shopping[i]
            self.shopping[i] = data
        print('数据获取成功')
    def sava(self):
        '保存'
        print('保存中...')
        # 创建数据集
        data = {'价格':self.prices,
                '店铺':self.shop,
                '店铺链接':self.shopping,
                '评论数/销量':self.ping
                }
        pd = pandas.DataFrame(data)

        # 写入文件
        pd.to_excel('JD data.xlsx',index = False)
        time.sleep(2)
        print('保存成功')

..........................................................................................................................................................

flowchart LR

A[开始] --> B[创建GetData类]

B --> C[访问京东iPhone4s搜索页面]

C --> D[点击微信登录]

D --> E[等待登录10秒]

E --> F[获取页面源代码]

F --> G[创建Sift类]

G --> H[解析前端页面数据]

H --> I[获取价格信息]

I --> J[获取店铺信息]

J --> K[获取店铺链接]

K --> L[获取评论数/销量]

L --> M[保存数据为Excel]

M --> N[结束]

..........................................................................................................................................................

可视化主函数：

..........................................................................................................................................................

import re
from wordcloud import WordCloud
import matplotlib.pyplot as plt
from collections import Counter

# 模拟的TXT文件内容
with open("唐卡吊坠2.txt","r") as f:
    txt_data =f.read()


# 清洗数据，去除特殊字符，并分词
words = re.findall(r'[\u4e00-\u9fa5]+', txt_data)  # 仅保留汉字

# 统计词频
word_counts = Counter(words)

# 绘制词云图
font_path = '方正仿宋简体.ttf'  # 字体路径，需要根据实际情况修改
wordcloud = WordCloud(font_path=font_path, width=800, height=400, background_color='white').generate_from_frequencies(word_counts)

# 显示词云图
plt.figure(figsize=(10, 5))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.show()

# 绘制柱形图
common_words = word_counts.most_common(10)
labels, values = zip(*common_words)

plt.figure(figsize=(10, 5))
plt.bar(labels, values)

# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei']  # 指定默认字体
plt.rcParams['axes.unicode_minus'] = False  # 解决保存图像时负号'-'显示为方块的问题

plt.xlabel('Words')
plt.ylabel('Count')
plt.title('Top 10 Most Common Words')
plt.xticks(rotation=45)  # 旋转x轴标签，以便更好地显示
plt.show()

..........................................................................................................................................................

flowchart LR

A[开始] --> B[读取TXT文件内容]

B --> C[清洗数据，去除特殊字符，并分词]

C --> D[统计词频]

D --> E[绘制词云图]

E --> F[显示词云图]

F --> G[获取最常见的10个词]

G --> H[绘制柱形图]

H --> I[显示柱形图]

I --> J[结束]

..........................................................................................................................................................

运行函数：

..........................................................................................................................................................

import get
import look

class Main:
    def __init__(self):
        # 获取目标数据
        geter = get.Sift()   # 创建get.py文件中的Sift类
        geter.take()
        geter.sava()
        # 进行可视化
        layout = look.MakePlot()
        layout.make()

if __name__ == '__main__':
    Main()

..........................................................................................................................................................

flowchart LR

A[开始] --> B[创建Main类]

B --> C[创建get.py中的Sift类]

C --> D[调用take()方法获取数据]

D --> E[调用sava()方法保存数据]

E --> F[创建look.py中的MakePlot类]

F --> G[调用make()方法进行可视化]

G --> H[结束]

...........................................................................................................................................................

总流程：

..........................................................................................................................................................

获取数据：

flowchart LR

A[开始] --> B[创建GetData类]

B --> C[访问京东iPhone4s搜索页面]

C --> D[点击微信登录]

D --> E[等待登录10秒]

E --> F[获取页面源代码]

F --> G[创建Sift类]

G --> H[解析前端页面数据]

H --> I[获取价格信息]

I --> J[获取店铺信息]

J --> K[获取店铺链接]

K --> L[获取评论数/销量]

L --> M[保存数据为Excel]

M --> N[结束]

...........................................................................................................................................................

可视化:

...........................................................................................................................................................

flowchart LR

A[开始] --> B[读取TXT文件内容]

B --> C[清洗数据，去除特殊字符，并分词]

C --> D[统计词频]

D --> E[绘制词云图]

E --> F[显示词云图]

F --> G[获取最常见的10个词]

G --> H[绘制柱形图]

H --> I[显示柱形图]

I --> J[结束]

...........................................................................................................................................................

运行：

...........................................................................................................................................................

flowchart LR

A[开始] --> B[创建Main类]

B --> C[创建get.py中的Sift类]

C --> D[调用take()方法获取数据]

D --> E[调用sava()方法保存数据]

E --> F[创建look.py中的MakePlot类]

F --> G[调用make()方法进行可视化]

G --> H[结束]

...........................................................................................................................................................

Guff_hys-CSDN博客

...........................................................................................................................................................