以下是使用Python编写的针对淘宝、天猫、京东详情页的爬虫实例。请注意，这些实例仅供参考，实际使用时可能需要根据网站结构的变化进行调整，并且需要遵守各平台的爬虫协议和法律法规。
淘宝详情页爬虫实例
-
环境准备：
- Python 3.x
- Selenium库
- ChromeDriver（或对应浏览器的WebDriver）
-
代码实现：
|---|-----------------------------------------------------------------------------------------------|
| | from selenium import webdriver |
| | from selenium.webdriver.common.by import By |
| | from selenium.webdriver.common.keys import Keys |
| | import time |
| | import csv |
| | |
| | # 初始化WebDriver |
| | driver = webdriver.Chrome() |
| | |
| | # 打开淘宝并搜索商品 |
| | driver.get('https://www.taobao.com/') |
| | driver.maximize_window() # 最大化浏览器窗口 |
| | driver.implicitly_wait(10) # 设置隐式等待时间 |
| | |
| | # 搜索商品（这里以"手机"为例） |
| | search_keyword = '手机' |
| | driver.find_element(By.XPATH, '//*[@id="q"]').send_keys(search_keyword) |
| | driver.find_element(By.XPATH, '//*[@id="J_TSearchForm"]/div[1]/button').click() |
| | |
| | # 等待搜索结果加载完成 |
| | time.sleep(5) |
| | |
| | # 解析搜索结果页面并提取商品详情页链接 |
| | product_links = [] |
| | for item in driver.find_elements(By.XPATH, '//div[@class="grid g-clearfix"]/div/div'): |
| | detail_url = item.find_element(By.XPATH, './/div[@class="pic"]/a').get_attribute('href') |
| | product_links.append(detail_url) |
| | |
| | # 遍历商品详情页链接并提取所需信息 |
| | with open('taobao_products.csv', 'w', newline='', encoding='utf-8') as csvfile: |
| | fieldnames = ['title', 'price', 'seller', 'location', 'detail_url'] |
| | writer = csv.DictWriter(csvfile, fieldnames=fieldnames) |
| | |
| | writer.writeheader() |
| | |
| | for link in product_links: |
| | driver.get(link) |
| | time.sleep(3) # 等待详情页加载完成 |
| | |
| | title = driver.find_element(By.XPATH, '//*[@id="J_DetailHeader"]/div[1]/h1').text |
| | price = driver.find_element(By.XPATH, '//*[@id="J_StrPrice"]/em').text |
| | seller = driver.find_element(By.XPATH, '//*[@id="J_OtherOptions"]/div[1]/p[1]/a').text |
| | location = driver.find_element(By.XPATH, '//*[@id="J_OtherOptions"]/div[1]/p[2]/span').text |
| | |
| | writer.writerow({ |
| | 'title': title, |
| | 'price': price, |
| | 'seller': seller, |
| | 'location': location, |
| | 'detail_url': link |
| | }) |
| | |
| | # 关闭WebDriver |
| | driver.quit() |
天猫详情页爬虫实例
- 环境准备：与淘宝相同。
- 代码实现（以搜索"羽毛球"为例）：
|---|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
| | from selenium import webdriver |
| | from selenium.webdriver.common.by import By |
| | from selenium.webdriver.support.ui import WebDriverWait |
| | from selenium.webdriver.support import expected_conditions as EC |
| | import time |
| | import csv |
| | |
| | # 初始化WebDriver |
| | driver = webdriver.Chrome() |
| | |
| | # 打开天猫并搜索商品 |
| | driver.get('https://list.tmall.com/') |
| | driver.maximize_window() |
| | driver.implicitly_wait(10) |
| | |
| | # 搜索商品（这里以"羽毛球"为例） |
| | search_keyword = '羽毛球' |
| | driver.get(f'https://list.tmall.com/search_product.htm?q={search_keyword}') |
| | |
| | # 等待搜索结果加载完成 |
| | wait = WebDriverWait(driver, 10) |
| | page_total_element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.tm-pagination .ui-page-item.ui-page-item-last em'))) |
| | page_total = page_total_element.text |
| | |
| | # 解析搜索结果页面并提取商品信息 |
| | product_info = [] |
| | for page in range(1, int(page_total) + 1): |
| | try: |
| | # 如果是非第一页，则进行翻页操作 |
| | if page > 1: |
| | input_element = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.ui-page > div.ui-page-wrap > b.ui-page-skip > form > input.ui-page-skipTo'))) |
| | input_element.clear() |
| | input_element.send_keys(page) |
| | submit_button = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '.ui-page > div.ui-page-wrap > b.ui-page-skip > form > button.ui-btn-s'))) |
| | submit_button.click() |
| | time.sleep(2) # 等待页面加载 |
| | |
| | # 提取当前页的商品信息 |
| | goods = driver.find_elements(By.CSS_SELECTOR, '#J_ItemList .product') |
| | for good in goods: |
| | title = good.find_element(By.CSS_SELECTOR, '.productTitle').text |
| | price = good.find_element(By.CSS_SELECTOR, '.productPrice').text.replace('¥', '') |
| | detail_url = good.find_element(By.CSS_SELECTOR, '.productImg').get_attribute('href') |
| | product_info.append({ |
| | 'title': title, |
| | 'price': price, |
| | 'detail_url': detail_url |
| | }) |
| | except Exception as e: |
| | print(f"Error on page {page}: {e}") |
| | |
| | # 将商品信息写入CSV文件 |
| | with open('tmall_products.csv', 'w', newline='', encoding='utf-8') as csvfile: |
| | fieldnames = ['title', 'price', 'detail_url'] |
| | writer = csv.DictWriter(csvfile, fieldnames=fieldnames) |
| | writer.writeheader() |
| | for product in product_info: |
| | writer.writerow(product) |
| | |
| | # 关闭WebDriver |
| | driver.quit() |
京东详情页爬虫实例
- 环境准备：与淘宝相同。
- 代码实现（以搜索"手机"为例，并提取详情页图片）：
|---|---------------------------------------------------------------------------------------------------------------------------------------|
| | from selenium import webdriver |
| | from selenium.webdriver.common.by import By |
| | from selenium.webdriver.support.ui import WebDriverWait |
| | from selenium.webdriver.support import expected_conditions as EC |
| | import time |
| | import os |
| | import requests |
| | |
| | # 初始化WebDriver |
| | driver = webdriver.Chrome() |
| | |
| | # 打开京东并搜索商品 |
| | driver.get('https://search.jd.com/') |
| | driver.maximize_window() |
| | driver.implicitly_wait(10) |
| | |
| | # 搜索商品（这里以"手机"为例） |
| | search_keyword = '手机' |
| | driver.find_element(By.XPATH, '//*[@id="key"]').send_keys(search_keyword) |
| | driver.find_element(By.XPATH, '//*[@id="search"]/div/button').click() |
| | |
| | # 等待搜索结果加载完成 |
| | wait = WebDriverWait(driver, 10) |
| | |
| | # 提取商品详情页链接并进入详情页提取图片 |
| | product_links = [] |
| | for item in driver.find_elements(By.CSS_SELECTOR, '.gl-item'): |
| | detail_url = item.find_element(By.CSS_SELECTOR, '.p-name em a').get_attribute('href') |
| | product_links.append(detail_url) |
| | |
| | headers = { |
| | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' |
| | } |
| | |
| | for link in product_links: |
| | driver.get(link) |
| | time.sleep(3) # 等待详情页加载完成 |
| | |
| | # 提取图片链接并下载 |
| | image_urls = [] |
| | try: |
| | images = driver.find_elements(By.CSS_SELECTOR, '.sku-gallery img') |
| | for img in images: |
| | image_urls.append(img.get_attribute('src')) |
| | except Exception as e: |
| | print(f"Error extracting images from {link}: {e}") |
| | continue |
| | |
| | image_dir = f'./jd_images/{link.split("/")[-1]}' |
| | if not os.path.exists(image_dir): |
| | os.makedirs(image_dir) |