爬虫获取数据：selenium的应用

当我们在爬取数据时，如：古诗网。有时会出现输出内容不全的情况，针对这种问题如何解决？

个人思路：在遍历网页内容时，如果未发现显示全部内容字样，说明该条数据内容完整，则立即输出。若识别到显示全部内容，则表示内容不全。需要click点击事件，跳转成功后获取该页面中我们主要的数据，例如古诗名、作者、古诗内容等。获取全部数据后返回初始页面，继续寻找显示全部内容。如此循环，直到遍历完成。

此处默认已掌握全部相关知识点，只提供代码。欢迎大家探讨。

复制代码

import time
import random
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
options = webdriver.EdgeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)
options = webdriver.EdgeOptions()
options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_experimental_option('useAutomationExtension', False)

# 加载当前浏览器驱动.exe.sh
edgeDriver = Service("../drivera/msedgedriver.exe")

# 实例化当前的浏览器对象
browser = webdriver.Edge(service=edgeDriver, options=options)

# 设置浏览器超时
wait = WebDriverWait(browser, 10)

browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
  'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})'
})
# 数据容器
poem_data = []
count = 1

url = "https://www.shicimingju.com/chaxun/zuozhe/46.html"
browser.get(url)
time.sleep(2)

# 获取所有诗词链接
more_links = browser.find_elements(By.CLASS_NAME, 'more')
total_links = len(more_links)
print(f"第一页找到 {total_links} 个诗词链接")

# 循环处理每个链接
for i, link in enumerate(more_links):
    try:
        print(f"\n--- 处理第 {count}/{total_links} 个链接 ---")

        # 滚动并点击链接
        browser.execute_script("arguments[0].scrollIntoView();", link)
        time.sleep(1)
        link.click()
        time.sleep(2)

        title = WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, "h1"))
        ).text.strip()

        author_elem = browser.find_elements(By.CSS_SELECTOR, "div.source, .name")
        author = "未知作者"
        if author_elem:
            author = author_elem[0].text.strip()
            author = author.split("·")[-1] if "·" in author else author

        # 提取内容
        content_elem = WebDriverWait(browser, 10).until(
            EC.presence_of_element_located((By.CLASS_NAME, "text"))
        )
        soup = BeautifulSoup(content_elem.get_attribute('innerHTML'), "html.parser")
        content = soup.get_text().strip()

        # 清理内容
        exclude = ["注释", "赏析", "作品赏析"]
        content = "\n".join([line for line in content.split("\n") if not any(x in line for x in exclude)])

        # 保存数据
        poem_data.append({
            "序号": count,
            "标题": title,
            "作者": author,
            "内容": content
        })


        print("=" * 50)
        print(f"【{title}】 - {author} \n{content}")

        count += 1

        # 返回上一页
        browser.back()
        time.sleep(1)
    finally:
        pass
#
# # 保存到Excel
# if poem_data:
#     pd.DataFrame(poem_data).to_excel("陆游诗词_简化版.xlsx", index=False)
#     print(f"\n成功保存 {len(poem_data)} 条数据")
#
# browser.quit()