根据xlsx文件第一列的网址爬虫

selenium+Xpath

在与该ipynb文件同文件下新增一个111.xlsx，第一列放一堆需要爬虫的同样式网页

然后使用selenium+Xpath爬虫

from selenium import webdriver
from selenium.webdriver.common.by import By
import openpyxl
import time

def crawl_data(driver, url):
    driver.get(url)
    time.sleep(5)  # 等待页面加载

    # 爬取指定的XPath内容
    xpath1 = '//*[@id="main"]/div/div/div[2]/div[1]/div/div[1]/div'
    xpath2 = '//*[@id="main"]/div/div/div[2]/div[1]/div/div[1]/p/span'
    xpath3 = '//*[@id="descriptionDiv"]/p'
    xpath4 = '//*[@id="introductionDiv"]/p'
    
    content1 = driver.find_element(By.XPATH, xpath1).text
    content2 = driver.find_element(By.XPATH, xpath2).text
    content3 = driver.find_element(By.XPATH, xpath3).text
    content4 = driver.find_element(By.XPATH, xpath4).text

    return content1, content2, content3, content4

# 启动Chrome浏览器
chrome_options = webdriver.ChromeOptions()
driver = webdriver.Chrome(options=chrome_options)

# 打开Excel文件
file_path = "111.xlsx"
wb = openpyxl.load_workbook(file_path)
ws = wb.active

# 获取第一列的网址
urls = [cell.value for cell in ws["A"]]

# 逐个处理每个网址
for i, url in enumerate(urls, start=1):
    print(f"处理第{i}个网址: {url}")
    
    url_str = str(url) if url is not None else ""
    
    # 爬取数据
    data1, data2, data3, data4 = crawl_data(driver, url_str)

    # 写回Excel文件
    ws.cell(row=i, column=2, value=data1)
    ws.cell(row=i, column=3, value=data2)
    ws.cell(row=i, column=4, value=data3)
    ws.cell(row=i, column=5, value=data4)

# 保存Excel文件
wb.save(file_path)

# 关闭浏览器
driver.quit()

使用Xpath方法相比css定位的好处：如果页面的层级结构非常复杂，XPath提供了更复杂的选择和过滤方式，可以更灵活地定位元素

相比id、class定位的好处：XPath可以根据元素的属性值进行定位，这在某些情况下很有用，尤其是在没有独特标识符（如ID）的情况下

同样使用selenium其它定位的方法

按ID、类名

# 以ID定位元素
element_by_id = driver.find_element(By.ID, 'articleContentId')
print("Element by ID:", element_by_id.text)

# 以类名定位元素
element_by_class = driver.find_element(By.CLASS_NAME, 'title-article')
print("Element by Class Name:", element_by_class.text)

按css选择器

# 以CSS选择器定位元素
element_by_css = driver.find_element(By.CSS_SELECTOR, '#articleContentId')
print("Element by CSS Selector:", element_by_css.text)

不用selenium

Requests + BeautifulSoup

使用BeautifulSoup时，通常通过选择器（CSS选择器）而不是XPath来提取元素

import requests
from bs4 import BeautifulSoup

url = "https://example.com"
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')

# 使用CSS选择器
title_element = soup.select_one('#articleContentId')
title = title_element.text if title_element else "Title not found"

paragraphs = soup.select('#articleContentId + div.content p')

# 输出结果
print(f"Title: {title}")
for paragraph in paragraphs:
    print(paragraph.text)

Scrapy

在Scrapy中，可以使用XPath或CSS选择器，具体取决于你的喜好或页面的结构。

XPath

import scrapy

class MySpider(scrapy.Spider):
    name = 'myspider'
    start_urls = ['https://example.com']

    def parse(self, response):
        # 使用XPath
        title = response.xpath('//*[@id="articleContentId"]/text()').get()
        paragraphs = response.xpath('//*[@id="articleContentId"]/following-sibling::div[@class="content"]/p/text()').getall()

        # 输出结果
        print(f"Title: {title}")
        for paragraph in paragraphs:
            print(paragraph)

css

import scrapy

class MySpider(scrapy.Spider):
    name = 'myspider'
    start_urls = ['https://example.com']

    def parse(self, response):
        # 使用CSS选择器
        title = response.css('#articleContentId::text').get()
        paragraphs = response.css('#articleContentId + div.content p::text').getall()

        # 输出结果
        print(f"Title: {title}")
        for paragraph in paragraphs:
            print(paragraph)

Pyppeteer / Playwright

使用Pyppeteer或Playwright时，你可以通过JavaScript在页面上执行脚本来获取数据，而不是直接使用XPath

XPath

from pyppeteer import launch

async def crawl_data(url):
    browser = await launch()
    page = await browser.newPage()
    await page.goto(url)

    # 使用JavaScript执行脚本获取内容
    title = await page.evaluate('document.evaluate("//*[@id=\"articleContentId\"]", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue.textContent')
    paragraphs = await page.evaluate('Array.from(document.evaluate("//*[@id=\"articleContentId\"]//p", document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null)).map(p => p.textContent)')

    # 输出结果
    print(f"Title: {title}")
    for paragraph in paragraphs:
        print(paragraph)

    await browser.close()

# 使用asyncio来运行异步代码
import asyncio
asyncio.get_event_loop().run_until_complete(crawl_data('https://example.com'))

element选择器

Element选择器实际上是通过HTML元素的标签名来选择元素，而不是通过ID或类名。

HTML元素的标签名就是元素的名称，通常是由尖括号括起来的部分。下述h1、p、a就是HTML元素的标签名

<h1>这是一个标题</h1>

<p>这是一个段落</p>

<a href="https://www.example.com">这是一个链接</a>

所以，如果你想使用element选择器，你可以这样修改代码：

from pyppeteer import launch

async def crawl_data(url):
    browser = await launch()
    page = await browser.newPage()
    await page.goto(url)

    # 使用JavaScript执行脚本获取内容
    title = await page.evaluate('document.querySelector("h1#articleContentId").textContent')
    paragraphs = await page.evaluate('Array.from(document.querySelector("h1#articleContentId + div.content").querySelectorAll("p")).map(p => p.textContent)')

    # 输出结果
    print(f"Title: {title}")
    for paragraph in paragraphs:
        print(paragraph)

    await browser.close()

# 使用asyncio来运行异步代码
import asyncio
asyncio.get_event_loop().run_until_complete(crawl_data('https://example.com'))

在这个例子中，我们使用了document.querySelector("h1#articleContentId")来选择ID为articleContentId的h1元素，以及document.querySelector("h1#articleContentId + div.content").querySelectorAll("p")来选择与该h1元素相邻的div元素内的所有p元素。

css

from pyppeteer import launch

async def crawl_data(url):
    browser = await launch()
    page = await browser.newPage()
    await page.goto(url)

    # 使用JavaScript执行脚本获取内容
    title = await page.evaluate('document.querySelector("#articleContentId").textContent')
    paragraphs = await page.evaluate('Array.from(document.querySelectorAll("#articleContentId p")).map(p => p.textContent)')

    # 输出结果
    print(f"Title: {title}")
    for paragraph in paragraphs:
        print(paragraph)

    await browser.close()

# 使用asyncio来运行异步代码
import asyncio
asyncio.get_event_loop().run_until_complete(crawl_data('https://example.com'))