python 并行请求多个 HTTP 接口

用一个不重复的昵称2025-02-18 21:08

在需要并行请求多个 HTTP 接口的场景中，并且每个接口的查询参数（query）都不同时，可以结合 Python 的 concurrent.futures.ThreadPoolExecutor 来实现高效的并行请求处理。下面是一个示例代码，展示如何为每个接口配置不同的查询参数，并执行并行请求：

import requests

from concurrent.futures import ThreadPoolExecutor, as_completed

定义接口的 URL 列表

urls = [

"http://1",

]

定义每个请求的 query 参数

queries = [

"query_1",

"query_2",

"query_3",

"query_4",

"query_5"

]

请求头

headers = {

"Content-Type": "application/json"

}

定义请求函数

python 复制代码

# 定义请求函数
def post_request(url, query):
    data = {
        "serviceVersion": "6",
        "request": {"query": query},
        "skipCache": True,
        "serviceId": "game-predicte-intention",
        "fetchResultNodeIds": ["12"]
    }

    try:
        response = requests.post(url, json=data, headers=headers)
        response.raise_for_status()  # 检查请求是否成功
        result = response.json().get('result', {})
        return url, result
    except requests.RequestException as e:
        return url, f"Request failed: {e}"

使用多线程进行并行请求

python 复制代码

# 使用多线程进行并行请求
def parallel_requests(urls, queries):
    results = {}
    with ThreadPoolExecutor(max_workers=len(urls)) as executor:
        # 提交每个 (url, query) 对应的请求
        future_to_request = {executor.submit(post_request, url, query): (url, query) for url, query in zip(urls, queries)}

        for future in as_completed(future_to_request):
            url, query = future_to_request[future]
            try:
                url, result = future.result()
                results[(url, query)] = result
                print(f"URL: {url}, Query: {query}, Result: {result}")
            except Exception as e:
                print(f"Error processing URL {url} with Query {query}: {e}")

    return results

执行并行请求

result_set = parallel_requests(urls, queries)