3 种 SERP 数据处理方案对比:流式 / 批处理 / 缓存

背景

调 serpbase 拿数据后怎么用?3 种处理模式:流式实时、批处理归档、缓存去重。

1. 流式(实时)

import requests
import json

def stream_serp_results(query, callback):
    """流式返回,每个结果立即回调"""
    r = requests.post(
        "https://api.serpbase.dev/google/search",
        headers={"X-API-Key": "sk_xxx"},
        json={"q": query, "gl": "us", "num": 20},
        timeout=10,
    )
    data = r.json()
    for i, item in enumerate(data.get("organic", []), 1):
        callback({
            "rank": i,
            "title": item["title"],
            "link": item["link"],
        })

# 用法
def my_callback(item):
    print(f"  #{item['rank']}: {item['title']}")

stream_serp_results("best serp api", my_callback)

优势 :实时、用户能立即看到

劣势:每次都调 API,成本高

2. 批处理(归档)

import schedule
import json
from datetime import datetime

def batch_serp_processing(queries, output_file):
    """批量拉 SERP 写 JSON 归档"""
    today = datetime.now().strftime("%Y-%m-%d")
    all_results = {"date": today, "results": []}

    for q in queries:
        try:
            r = requests.post(
                "https://api.serpbase.dev/google/search",
                headers={"X-API-Key": "sk_xxx"},
                json={"q": q, "gl": "us", "num": 10},
                timeout=10,
            )
            all_results["results"].append({
                "query": q,
                "data": r.json(),
            })
        except Exception as e:
            print(f"{q} failed: {e}")

    # 写 JSON 归档
    with open(output_file, "w") as f:
        json.dump(all_results, f, ensure_ascii=False, indent=2)

# 每天跑 1 次
schedule.every().day.at("03:00").do(
    batch_serp_processing,
    ["SERP API", "cheap SERP API", "SERP API 选型"],
    "archive/2026-06-19.json"
)

优势 :成本低(一次拉所有)

劣势:数据延迟(下次拉要等 24 小时)

3. 缓存(去重)

import redis
import json
import hashlib

r = redis.Redis()
CACHE_TTL = 300  # 5 分钟

def cached_serp_search(query, gl="us"):
    """带缓存的 SERP 查询"""
    cache_key = f"serp:{hashlib.md5(f'{query}|{gl}'.encode()).hexdigest()}"

    cached = r.get(cache_key)
    if cached:
        return json.loads(cached)

    r = requests.post(
        "https://api.serpbase.dev/google/search",
        headers={"X-API-Key": "sk_xxx"},
        json={"q": query, "gl": gl, "num": 5},
        timeout=10,
    )
    data = r.json()

    r.setex(cache_key, CACHE_TTL, json.dumps(data))
    return data

优势 :重复查询快、便宜

劣势:首次查询慢、cache miss 需要查 API

4. 3 种方案对比

维度	流式	批处理	缓存
实时性	立即	24h+ 延迟	5 分钟延迟
成本(1000 queries)	$0.30	$0.30	$0.10(50% 命中)
实现复杂度	★	★★	★★★
适用场景	实时用户查询	历史归档 / 监控	高频重复查询

5. 选型

场景	推荐
实时用户查询(LLM agent / ChatBot)	流式 + 缓存
SEO 监控 / 报告	批处理(每天 1 次)
高频 query(同一问题反复)	缓存(命中率高)
历史分析 / 回溯	批处理 + 数据库

6. 实战代码(综合 3 种)

class SerpProcessor:
    def __init__(self, cache_ttl=300):
        self.cache = redis.Redis()
        self.cache_ttl = cache_ttl

    def get(self, query, gl="us", source="realtime"):
        """综合入口:cache → batch → realtime"""

        # 1. cache hit(免费)
        cached = self.cache_get(query, gl)
        if cached:
            return cached

        # 2. batch(每日归档)
        if source == "batch":
            return self.batch_get(query, gl)

        # 3. realtime(API)
        return self.realtime_get(query, gl)

    def cache_get(self, query, gl):
        key = f"serp:{hash(f'{query}|{gl}')}"
        cached = self.cache.get(key)
        return json.loads(cached) if cached else None

    def batch_get(self, query, gl):
        # 从 Parquet / 数据库读
        df = pd.read_parquet("archive/*.parquet")
        row = df[(df["query"] == query) & (df["gl"] == gl)].sort_values("date").iloc[-1]
        return row.to_dict()

    def realtime_get(self, query, gl):
        # 调 API
        r = requests.post(
            "https://api.serpbase.dev/google/search",
            headers={"X-API-Key": "sk_xxx"},
            json={"q": query, "gl": gl, "num": 5},
            timeout=10,
        )
        data = r.json()
        # 写 cache
        key = f"serp:{hash(f'{query}|{gl}')}"
        self.cache.setex(key, self.cache_ttl, json.dumps(data))
        return data

7. 5 个工程细节

细节 1:cache key 设计

# 错:只 hash query
key = f"serp:{query}"

# 对:含所有影响结果的参数
key = f"serp:{hash(f'{query}|{gl}|{hl}|{num}')}"

细节 2:cache TTL 不要一刀切

def ttl_for_query(query):
    if "news" in query or "today" in query:
        return 60  # 1 分钟
    if "rank" in query:
        return 3600  # 1 小时
    return 300  # 默认 5 分钟

细节 3:batch 并发

import concurrent.futures

def batch_process_concurrent(queries, max_workers=10):
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex:
        futures = {ex.submit(self.realtime_get, q, gl): q for q in queries}
        for f in concurrent.futures.as_completed(futures):
            try:
                yield f.result()
            except Exception as e:
                print(f"{futures[f]} failed: {e}")

细节 4:cache 击穿防护

def safe_get(self, query, gl):
    key = f"serp:{hash(f'{query}|{gl}')}"
    cached = self.cache.get(key)
    if cached:
        return json.loads(cached)

    # 加锁防击穿
    lock_key = f"lock:{key}"
    if not self.cache.set(lock_key, "1", ex=5, nx=True):
        # 其他请求在回源
        time.sleep(0.1)
        return self.safe_get(query, gl)

    try:
        data = self.realtime_get(query, gl)
        self.cache.setex(key, self.cache_ttl, json.dumps(data))
        return data
    finally:
        self.cache.delete(lock_key)

细节 5:batch + cache 组合

def get_with_2layer_cache(self, query, gl="us"):
    # L1:process cache
    if query in self.process_cache:
        return self.process_cache[query]

    # L2:Redis cache
    cached = self.cache_get(query, gl)
    if cached:
        self.process_cache[query] = cached
        return cached

    # API
    data = self.realtime_get(query, gl)
    self.process_cache[query] = data
    return data

8. 实战数据(我项目 30 天)

指标	数值
总查询	100,000
缓存命中	60%
流式(实时)	40%
批处理	20%
cache 重复	40%
SerpBase 月成本	$12
节省	60%

小结

3 种数据处理选型:

• 流式:实时用户查询 + LLM agent
• 批处理:历史归档 + 监控 + 报告
• 缓存:高频重复查询 + 高 QPS

serpbase + 3 种组合,60% 缓存命中,月成本 12(纯流式 30),节省 60%。

serpbase auto-refund 100% 触发,cache miss 失败不扣 credit,放心加多层缓存。