技术背景
在电商数据采集领域，我们经常面临以下技术挑战：
- 反爬虫机制日益复杂，需要动态IP轮换和请求头伪装
- 页面结构频繁变化，传统的XPath解析容易失效
- 大规模并发请求的负载均衡和容错处理
- 数据一致性和完整性验证
Pangolin API技术架构分析
基于我们的实际使用经验，Pangolin API在技术架构上有以下优势：
1. 分布式爬虫集群
bash
# Pangolin API的分布式架构示意
┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐
│ Load Balancer │────│ Crawler Node 1 │────│ Data Parser │
│ │ │ │ │ │
│ 智能负载均衡 │────│ Crawler Node 2 │────│ 结构化处理 │
│ │ │ │ │ │
│ 请求分发 │────│ Crawler Node N │────│ 质量验证 │
└─────────────────┘ └─────────────────┘ └─────────────────┘2. 实际集成代码示例
python
import requests
import json
from typing import Dict, List, Optional
class PangolinAPIClient:
def __init__(self, api_key: str, base_url: str = "https://api.pangolinfo.com"):
self.api_key = api_key
self.base_url = base_url
self.session = requests.Session()
self.session.headers.update({
'Authorization': f'Bearer {api_key}',
'Content-Type': 'application/json'
})
def scrape_amazon_search(self, keyword: str, marketplace: str = "US",
page: int = 1) -> Dict:
"""
抓取亚马逊搜索结果页面
Args:
keyword: 搜索关键词
marketplace: 市场区域 (US, UK, DE, etc.)
page: 页码
Returns:
Dict: 包含商品列表和广告位信息的结构化数据
"""
endpoint = f"{self.base_url}/v1/amazon/search"
payload = {
"keyword": keyword,
"marketplace": marketplace,
"page": page,
"include_sponsored": True, # 包含Sponsored广告位
"output_format": "json"
}
try:
response = self.session.post(endpoint, json=payload)
response.raise_for_status()
return response.json()
except requests.exceptions.RequestException as e:
print(f"API请求失败: {e}")
return {}
def scrape_product_details(self, asin: str, marketplace: str = "US") -> Dict:
"""
抓取商品详情页面
Args:
asin: 商品ASIN
marketplace: 市场区域
Returns:
Dict: 商品详细信息
"""
endpoint = f"{self.base_url}/v1/amazon/product"
payload = {
"asin": asin,
"marketplace": marketplace,
"include_reviews": True,
"include_qa": True,
"output_format": "json"
}
try:
response = self.session.post(endpoint, json=payload)
response.raise_for_status()
return response.json()
except requests.exceptions.RequestException as e:
print(f"API请求失败: {e}")
return {}
# 使用示例
def main():
# 初始化客户端
client = PangolinAPIClient(api_key="your_api_key_here")
# 批量抓取搜索结果
keywords = ["wireless earbuds", "bluetooth speaker", "phone case"]
for keyword in keywords:
print(f"正在抓取关键词: {keyword}")
search_data = client.scrape_amazon_search(keyword)
if search_data and 'products' in search_data:
print(f"找到 {len(search_data['products'])} 个商品")
# 抓取前5个商品的详细信息
for product in search_data['products'][:5]:
asin = product.get('asin')
if asin:
detail_data = client.scrape_product_details(asin)
print(f"商品 {asin} 详情抓取完成")
if __name__ == "__main__":
main()3. 错误处理和重试机制
python
import time
from functools import wraps
def retry_on_failure(max_retries: int = 3, delay: float = 1.0):
"""
装饰器：API调用失败时自动重试
"""
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
for attempt in range(max_retries):
try:
return func(*args, **kwargs)
except Exception as e:
if attempt == max_retries - 1:
raise e
print(f"第 {attempt + 1} 次尝试失败: {e}")
time.sleep(delay * (2 ** attempt)) # 指数退避
return None
return wrapper
return decorator
class RobustPangolinClient(PangolinAPIClient):
@retry_on_failure(max_retries=3, delay=1.0)
def scrape_with_retry(self, endpoint: str, payload: Dict) -> Dict:
"""
带重试机制的API调用
"""
response = self.session.post(endpoint, json=payload)
response.raise_for_status()
return response.json()性能优化实践
1. 并发控制
python
import asyncio
import aiohttp
from typing import List
class AsyncPangolinClient:
def __init__(self, api_key: str, max_concurrent: int = 10):
self.api_key = api_key
self.semaphore = asyncio.Semaphore(max_concurrent)
async def scrape_batch_products(self, asins: List[str]) -> List[Dict]:
"""
批量异步抓取商品信息
"""
async with aiohttp.ClientSession() as session:
tasks = [self._scrape_single_product(session, asin) for asin in asins]
results = await asyncio.gather(*tasks, return_exceptions=True)
return [r for r in results if not isinstance(r, Exception)]
async def _scrape_single_product(self, session: aiohttp.ClientSession,
asin: str) -> Dict:
async with self.semaphore: # 控制并发数
# API调用逻辑
pass2. 数据缓存策略
python
import redis
import pickle
from datetime import timedelta
class CachedPangolinClient(PangolinAPIClient):
def __init__(self, api_key: str, redis_host: str = "localhost"):
super().__init__(api_key)
self.redis_client = redis.Redis(host=redis_host, decode_responses=False)
def get_cached_or_fetch(self, cache_key: str, fetch_func,
cache_ttl: int = 3600) -> Dict:
"""
缓存优先的数据获取策略
"""
# 尝试从缓存获取
cached_data = self.redis_client.get(cache_key)
if cached_data:
return pickle.loads(cached_data)
# 缓存未命中，调用API
fresh_data = fetch_func()
if fresh_data:
# 存入缓存
self.redis_client.setex(
cache_key,
cache_ttl,
pickle.dumps(fresh_data)
)
return fresh_data常见问题和解决方案
Q1: API调用频率限制怎么处理？
A: Pangolin API内置了智能频率控制，但建议在客户端也实现令牌桶算法：
python
import time
from threading import Lock
class RateLimiter:
def __init__(self, max_calls: int, time_window: int):
self.max_calls = max_calls
self.time_window = time_window
self.calls = []
self.lock = Lock()
def acquire(self):
with self.lock:
now = time.time()
# 清理过期的调用记录
self.calls = [call_time for call_time in self.calls
if now - call_time < self.time_window]
if len(self.calls) < self.max_calls:
self.calls.append(now)
return True
return FalseQ2: 如何处理数据解析失败？
A: 建立多层验证机制：
python
def validate_product_data(data: Dict) -> bool:
"""
验证商品数据完整性
"""
required_fields = ['asin', 'title', 'price', 'rating']
for field in required_fields:
if field not in data or not data[field]:
return False
# 数据类型验证
if not isinstance(data['price'], (int, float)):
return False
if not (0 <= data['rating'] <= 5):
return False
return True项目地址
完整的示例代码已上传到GitHub：
标签： Python API 数据采集 电商 爬虫 异步编程