Selenium 获取请求响应

复制代码
'''
Python 3.7
selenium==3.141.0
urllib3==1.26.2
Chromium 109.0.5405.0 (32 位) 
'''
python 复制代码
import json
from selenium import webdriver
from selenium.common.exceptions import WebDriverException
import time

options = webdriver.ChromeOptions()
# 谷歌浏览器位置
chrome_location = r'D:\\Program Files (x86)\\Google\Chrome\\Application\\chrome.exe'
# 谷歌浏览器驱动地址
chromedriver_path = r'D:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver_win32\\chromedriver.exe'

options.binary_location = chrome_location
###################################################################################
# 写法一
# (网上还有其他的方法,有的会报错 可能是版本问题,selenium.common.exceptions.InvalidArgumentException: Message: invalid argument: log type 'performance' not found,
# 下面两种测试正常)

caps = {
    'browserName': 'chrome',
    'version': '',
    'platform': 'ANY',
    'goog:loggingPrefs': {'performance': 'ALL'},
    'goog:chromeOptions': {'extensions': [], 'args': ['--headless']}
}

caps = {
    "browserName": "chrome",
    'goog:loggingPrefs': {'performance': 'ALL'}
}

driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps)
###################################################################################

###################################################################################
# 写法二 (建议用这种,selenium 4 测试也行)
# options.set_capability("goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"})
# driver = webdriver.Chrome(executable_path=chromedriver_path, options=options)
###################################################################################

# 查询的 IP
list_query = ['135.89.67.33', '34.66.45.22']

for query in list_query:
    driver.get(f'http://ip-api.com/json/{query}')
    # 等待所有请求完成,可以用等待界面元素方法
    time.sleep(10)
    logs = driver.get_log("performance")

    for item in logs:
        # print(item)
        log = json.loads(item["message"])["message"]
        # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        # pprint(log)
        if log["method"] == 'Network.responseReceived':
            url = log['params']['response']['url']
            if url == 'data:,':  # 过滤掉初始data页面,后续可以根据 log['params']['response']['type']过滤请求类型
                continue
            print('请求', url)
            request_id = log['params']['requestId']
            response_headers = log['params']['response']['headers']
            status_code = log['params']['response']['status']
            try:
                request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id})
            except WebDriverException:  # 没有后台数据获取时会有异常
                request_data = None

            response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body']
            print('响应', response_body)


'''
输出:
请求 http://ip-api.com/json/135.89.67.33
响应 {"status":"success","country":"United States","countryCode":"US","region":"IN","regionName":"Indiana","city":"Indianapolis","zip":"46204","lat":39.7709,"lon":-86.1585,"timezone":"America/Indiana/Indianapolis","isp":"AT\u0026T Services","org":"AT\u0026T Services, Inc.","as":"","query":"135.89.67.33"}

请求 http://ip-api.com/json/34.66.45.22
响应 {"status":"success","country":"United States","countryCode":"US","region":"IA","regionName":"Iowa","city":"Council Bluffs","zip":"","lat":41.2619,"lon":-95.8608,"timezone":"America/Chicago","isp":"Google LLC","org":"Google Cloud (us-central1)","as":"AS396982 Google LLC","query":"34.66.45.22"}
'''
复制代码
'''
Python 3.8
selenium==4.21.0
urllib3==2.2.2
Chromium 109.0.5405.0 (32 位) 
'''
python 复制代码
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import WebDriverException
import json
import time

# 谷歌浏览器位置
chrome_location = r'D:\\Program Files (x86)\\Google\Chrome\\Application\\chrome.exe'
# 谷歌浏览器驱动地址
chromedriver_path = r'D:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver_win32\\chromedriver.exe'

# 启用性能日志
options = Options()
options.set_capability("goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"})
options.binary_location = chrome_location
# 启动WebDriver
service = Service(executable_path=chromedriver_path)
driver = webdriver.Chrome(service=service, options=options)

# 查询的 IP
list_query = ['135.89.67.33', '34.66.45.22']

for query in list_query:
    driver.get(f'http://ip-api.com/json/{query}')
    # 等待所有请求完成,可以用等待界面元素方法
    time.sleep(10)
    logs = driver.get_log("performance")

    for item in logs:
        # print(item)
        log = json.loads(item["message"])["message"]
        # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        # pprint(log)
        if log["method"] == 'Network.responseReceived':
            url = log['params']['response']['url']
            if url == 'data:,':  # 过滤掉初始data页面,后续可以根据 log['params']['response']['type']过滤请求类型
                continue
            print('请求', url)
            request_id = log['params']['requestId']
            response_headers = log['params']['response']['headers']
            status_code = log['params']['response']['status']
            try:
                request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id})
            except WebDriverException:  # 没有后台数据获取时会有异常
                request_data = None

            response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body']
            print('响应', response_body)
复制代码
'''
参考:
https://blog.csdn.net/MXB_1220/article/details/131775148
https://blog.csdn.net/u014376732/article/details/133973141
https://www.cnblogs.com/szyicol/p/18093390
https://www.cnblogs.com/superhin/p/15023302.html
https://segmentfault.com/q/1010000043296964
'''
相关推荐
特立独行的猫A6 小时前
别再用 nohup 了:让应用在后台常驻的正确姿势与 systemd 服务详解
java·后端·python
吃我一窝jio9 小时前
Python与Go线程模型深度对比
java·开发语言·python
zwd200510 小时前
Manim 怎么放大画面的某一部分?
python·动画·可视化·manim·数学动画
屿华AI10 小时前
AUTOSAR 实战-诊断工具 | UDS 0x27 安全访问的配置化实现:种子密钥算法插件化设计
python·测试工具·汽车
RobinDevNotes11 小时前
Claude Code 剪视频,不上传、不要 API Key
人工智能·python·音视频·开发工具·ipython
如夏倾城12 小时前
VRRP(虚拟路由器冗余协议Virtual Router Redundancy Protocol)
网络·python·网络协议·安全·智能路由器·负载均衡·信息与通信
IvanCodes12 小时前
LangChain(04):Runnable 与 LCEL 的执行与组合机制
python·langchain·agent
特创数字科技13 小时前
纯本地离线运行的文档脱敏工具(技术分享)
前端·驱动开发·python
东莞市云毅网络有限公司14 小时前
用 Playwright 搭一套问答平台引用监测的最小骨架
python·数据清洗·rag·企业知识库·文档解析
Thanks_ks14 小时前
【第 010 讲】全面掌握 Python 变量与常量:变量的定义与赋值 | 标识符与命名规范 | 变量的底层机制 | 常量与魔术数字
python·内存·变量·标识符·常量·赋值·命名规范