永城信息网爬虫

永城信息网爬虫

一个小地方的信息网站竟然蕴含着大技术，该开发者把数据都藏在js中，对于爬虫新手来说还是非常有难度的，废话不都说，上代码

import requests
import json
import re
import os
import time

LIST_API = "https://www.ycxinxi.com/new/api/v1/infoList"
DETAIL_API = "https://www.ycxinxi.com/new/api/v1/infoDetail"

headers = {
    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
}

proxies = {
    'http': os.environ.get('HTTP_PROXY') or os.environ.get('http_proxy'),
    'https': os.environ.get('HTTPS_PROXY') or os.environ.get('https_proxy')
}
proxies = {k: v for k, v in proxies.items() if v}

try:
    # 获取列表
    params = {"cat": 3, "page": 2, "callback": "jsonp_callback"}
    res = requests.get(LIST_API, params=params, headers=headers, timeout=10, proxies=proxies or None, verify=False)
    json_str = re.search(r'jsonp_callback\((.*)\)', res.text).group(1)
    data = json.loads(json_str)

    if data.get('code') == '00':
        jobs = data.get('data', {}).get('list', [])
        print(f"共找到 {len(jobs)} 条招聘信息\n")

        for i, job in enumerate(jobs, 1):
            job_id = job.get('id')
            print(f"\n{'='*80}")
            print(f"第 {i} 条 标题：{job.get('title')}招聘详情 (ID: {job_id})")
            print('='*80)

            # 获取详情
            detail_params = {"id": job_id, "callback": "jsonp_callback"}
            detail_res = requests.get(DETAIL_API, params=detail_params, headers=headers, timeout=10, proxies=proxies or None, verify=False)
            detail_json = re.search(r'jsonp_callback\((.*)\)', detail_res.text).group(1)
            detail_data = json.loads(detail_json)

            if detail_data.get('code') == '00':
                detail = detail_data.get('data', {})
                for key, value in detail.items():
                    print(f"{key}: {value}")

            time.sleep(0.5)  # 避免请求过快

except Exception as e:
    print(f"❌ 错误: {e}")

该代码仅供学习使用，爬虫有风险，操作需谨慎