瑞数案例欧冶解读py和js文件最终版

py文件解读

首先文件先呈上

import json
import time

import requests
from lxml import etree
from loguru import logger
import execjs

session = requests.Session()

headers = {
    "Accept": "*/*",
    "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
    "Origin": "https://www.ouyeel.com",
    "Referer": "https://www.ouyeel.com/search-ng/queryResource/index",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
}

list_url = "https://www.ouyeel.com/search-ng/commoditySearch/queryCommodityResult"


def fetch_1(): # 用于获取ts，auto，env三件套！ 同时也保存了cookie1，所以不用管cookie1
    response = session.post(list_url, headers=headers)
    tree = etree.HTML(response.text)
    meta_content = tree.xpath('//meta[2]/@content')[0]
    ts_js = tree.xpath('//script[1]/text()')[0]
    auto_script_url = 'https://www.ouyeel.com' + tree.xpath('//script[@src]/@src')[0]
    auto_script_js = session.get(auto_script_url).text

    return meta_content, ts_js, auto_script_js

def gen_cookie2(meta_content, ts_js, auto_script_js):
    with open("ouyeel.js", encoding='utf-8') as fp: # 这个js文件只有上节课补环境的部分，只缺三件套
        js_code = fp.read()
    js_code = js_code.replace("meta_content", meta_content).replace("'ts_js'", ts_js).replace("'auto_script_js'",
                                                                                              auto_script_js)
    js_compile = execjs.compile(js_code)
    js_cookie = js_compile.call("get_cookies")
    cookie_key_value = js_cookie.split(";")[0].split("=")
    session.cookies.update({cookie_key_value[0]: cookie_key_value[1]})


def fetch_2(page_index): # 填写第几页即可
    data = {
        "criteriaJson": '{"pageIndex":%d,"pageSize":50,"industryComponent":null,"channel":null,"productType":null,'
                        '"sort":null,"warehouseCode":null,"key_search":null,"is_central":null,"searchField":null,'
                        '"companyCode":null,"inquiryCategory":null,"inquirySpec":null,"provider":null,'
                        '"packCodes":null,"shopCode":null,"steelFactory":null,"resourceIds":null,"jsonParam":{},'
                        '"excludeShowSoldOut":null}' % page_index
    }
    response = session.post(list_url, headers=headers, data=data)
    logger.info(f"第{page_index}页产品列表: ", response, response.text)
    ret_list_str = response.json().get("resultList")
    ret_list = json.loads(ret_list_str)
    for item in ret_list:
        print(item)


def main():

    # for page in range(1,2):
        # (1) 发起第一次请求，获取cookie1以及源码三件套：meta_content, ts_js, auto_script_js
        meta_content, ts_js, auto_script_js = fetch_1()
        # print(session.cookies.get_dict())
        # (2) 基于源码三件套制作cookie2
        gen_cookie2(meta_content, ts_js, auto_script_js)
        # (3) 发起第二次请求，获取数据
        fetch_2(1)
        session.cookies.clear()
        time.sleep(1)


if __name__ == '__main__':
    main()

main方法追踪流程

获取三件套

替换js文件，将三件套写入同一个js文件中