Selenium 获取请求响应

复制代码
'''
Python 3.7
selenium==3.141.0
urllib3==1.26.2
Chromium 109.0.5405.0 (32 位) 
'''
python 复制代码
import json
from selenium import webdriver
from selenium.common.exceptions import WebDriverException
import time

options = webdriver.ChromeOptions()
# 谷歌浏览器位置
chrome_location = r'D:\\Program Files (x86)\\Google\Chrome\\Application\\chrome.exe'
# 谷歌浏览器驱动地址
chromedriver_path = r'D:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver_win32\\chromedriver.exe'

options.binary_location = chrome_location
###################################################################################
# 写法一
# (网上还有其他的方法,有的会报错 可能是版本问题,selenium.common.exceptions.InvalidArgumentException: Message: invalid argument: log type 'performance' not found,
# 下面两种测试正常)

caps = {
    'browserName': 'chrome',
    'version': '',
    'platform': 'ANY',
    'goog:loggingPrefs': {'performance': 'ALL'},
    'goog:chromeOptions': {'extensions': [], 'args': ['--headless']}
}

caps = {
    "browserName": "chrome",
    'goog:loggingPrefs': {'performance': 'ALL'}
}

driver = webdriver.Chrome(executable_path=chromedriver_path, options=options, desired_capabilities=caps)
###################################################################################

###################################################################################
# 写法二 (建议用这种,selenium 4 测试也行)
# options.set_capability("goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"})
# driver = webdriver.Chrome(executable_path=chromedriver_path, options=options)
###################################################################################

# 查询的 IP
list_query = ['135.89.67.33', '34.66.45.22']

for query in list_query:
    driver.get(f'http://ip-api.com/json/{query}')
    # 等待所有请求完成,可以用等待界面元素方法
    time.sleep(10)
    logs = driver.get_log("performance")

    for item in logs:
        # print(item)
        log = json.loads(item["message"])["message"]
        # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        # pprint(log)
        if log["method"] == 'Network.responseReceived':
            url = log['params']['response']['url']
            if url == 'data:,':  # 过滤掉初始data页面,后续可以根据 log['params']['response']['type']过滤请求类型
                continue
            print('请求', url)
            request_id = log['params']['requestId']
            response_headers = log['params']['response']['headers']
            status_code = log['params']['response']['status']
            try:
                request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id})
            except WebDriverException:  # 没有后台数据获取时会有异常
                request_data = None

            response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body']
            print('响应', response_body)


'''
输出:
请求 http://ip-api.com/json/135.89.67.33
响应 {"status":"success","country":"United States","countryCode":"US","region":"IN","regionName":"Indiana","city":"Indianapolis","zip":"46204","lat":39.7709,"lon":-86.1585,"timezone":"America/Indiana/Indianapolis","isp":"AT\u0026T Services","org":"AT\u0026T Services, Inc.","as":"","query":"135.89.67.33"}

请求 http://ip-api.com/json/34.66.45.22
响应 {"status":"success","country":"United States","countryCode":"US","region":"IA","regionName":"Iowa","city":"Council Bluffs","zip":"","lat":41.2619,"lon":-95.8608,"timezone":"America/Chicago","isp":"Google LLC","org":"Google Cloud (us-central1)","as":"AS396982 Google LLC","query":"34.66.45.22"}
'''
复制代码
'''
Python 3.8
selenium==4.21.0
urllib3==2.2.2
Chromium 109.0.5405.0 (32 位) 
'''
python 复制代码
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.common.exceptions import WebDriverException
import json
import time

# 谷歌浏览器位置
chrome_location = r'D:\\Program Files (x86)\\Google\Chrome\\Application\\chrome.exe'
# 谷歌浏览器驱动地址
chromedriver_path = r'D:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver_win32\\chromedriver.exe'

# 启用性能日志
options = Options()
options.set_capability("goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"})
options.binary_location = chrome_location
# 启动WebDriver
service = Service(executable_path=chromedriver_path)
driver = webdriver.Chrome(service=service, options=options)

# 查询的 IP
list_query = ['135.89.67.33', '34.66.45.22']

for query in list_query:
    driver.get(f'http://ip-api.com/json/{query}')
    # 等待所有请求完成,可以用等待界面元素方法
    time.sleep(10)
    logs = driver.get_log("performance")

    for item in logs:
        # print(item)
        log = json.loads(item["message"])["message"]
        # if "Network.response" in log["method"] or "Network.request" in log["method"] or "Network.webSocket" in log["method"]:
        # pprint(log)
        if log["method"] == 'Network.responseReceived':
            url = log['params']['response']['url']
            if url == 'data:,':  # 过滤掉初始data页面,后续可以根据 log['params']['response']['type']过滤请求类型
                continue
            print('请求', url)
            request_id = log['params']['requestId']
            response_headers = log['params']['response']['headers']
            status_code = log['params']['response']['status']
            try:
                request_data = driver.execute_cdp_cmd('Network.getRequestPostData', {'requestId': request_id})
            except WebDriverException:  # 没有后台数据获取时会有异常
                request_data = None

            response_body = driver.execute_cdp_cmd('Network.getResponseBody', {'requestId': request_id})['body']
            print('响应', response_body)
复制代码
'''
参考:
https://blog.csdn.net/MXB_1220/article/details/131775148
https://blog.csdn.net/u014376732/article/details/133973141
https://www.cnblogs.com/szyicol/p/18093390
https://www.cnblogs.com/superhin/p/15023302.html
https://segmentfault.com/q/1010000043296964
'''
相关推荐
m0_7231402315 分钟前
Python训练营-Day49
开发语言·python
北风toto1 小时前
python学习DataFrame数据结构
数据结构·python·学习
亿牛云爬虫专家1 小时前
微服务化采集平台:可扩展性与容错机制
python·微服务·架构·爬虫代理·扩展性·新浪财经·财经新闻
傻啦嘿哟1 小时前
Python爬虫动态IP代理报错全解析:从问题定位到实战优化
爬虫·python·tcp/ip
mit6.8241 小时前
[Meetily后端框架] Whisper转录服务器 | 后端服务管理脚本
c++·人工智能·后端·python
zhangfeng11331 小时前
python 数据分析 单细胞测序数据分析 相关的图表,常见于肿瘤免疫微环境、细胞亚群功能研究 ,各图表类型及逻辑关系如下
开发语言·python·数据分析·医学
柠檬豆腐脑1 小时前
Trae-Agent 内置工具深度解析
python·llm·agent
好奇的菜鸟2 小时前
在 Postman 中高效生成随机环境变量的完整指南
测试工具·lua·postman
ydl11282 小时前
机器学习基础知识【 激活函数、损失函数、优化器、 正则化、调度器、指标函数】
python·机器学习
车载测试工程师2 小时前
汽车功能安全-嵌入式软件测试(软件合格性测试)【目的、验证输入、集成&验证要求】11
功能测试·网络协议·测试工具·安全·车载系统·汽车·测试覆盖率