自己写的爬虫小案例

努力学习各种软件2024-04-28 22:56
网址：aHR0cDovL2pzc2NqZ3B0Lmp4d3JkLmdvdi5jbi8/dXJsPS92aWV3L3dvcmtpbmdVbml0L3dvcmtpbmdVbml0Lmh0bWw=
这串代码能够爬取勘察单位企业的详细信息。
python 复制代码
import requests
import time
import csv
f = open('勘察单位公司信息.csv','w',encoding='utf-8',newline='')
csv_writer = csv.DictWriter(f,fieldnames=[
    '公司名称',
    '涉及领域',
    '所属省市',
    '注册资本（万元）',
    '成立日期',
    '发证机关',
    '统一社会信用码',
    '经营范围',
    '单位网址',
    '法人',
    '主营业务',
    '注册地址',
    '证书网址',
    
])
csv_writer.writeheader()
cookies = {
    'JSESSIONID': 'BD8F06872E4D854606108A0F4AF55D79',
}
def get_response(id):
    info_url = f'http://jsscjgpt.jxwrd.gov.cn/api/employer/findByEmployer?unitId={id}'
    result = requests.get(url=info_url,headers=headers).json()
    return result
headers = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Language': 'zh-CN,zh;q=0.9',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Content-Type': 'application/json',
    # 'Cookie': 'JSESSIONID=BD8F06872E4D854606108A0F4AF55D79',
    'Origin': 'http://jsscjgpt.jxwrd.gov.cn',
    'Pragma': 'no-cache',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
}
for page in range(1,299):
    time.sleep(1)
    json_data = {
        'pageNum': page,
        'pageSize': 10,
        'query': '1',
        'unitName': '',
        'unitCode': '',
        'userName': '',
    }
    response = requests.post(
        'http://jsscjgpt.jxwrd.gov.cn/api/employer/findAll',
        cookies=cookies,
        headers=headers,
        json=json_data,
        verify=False,
    )
    json_response = response.json()
    for index in json_response['data']['list']:
        id = index['companyId']

        data = get_response(id)['data']
        dit = {
            '公司名称': data['name'],
            '涉及领域': data['categoryType'],
            '所属省市': data['address'],
            '注册资本（万元）': data['registeredMoney'],
            '成立日期': data['setUpYear'],
            '发证机关': data['certificateCompany'],
            '统一社会信用码': data['code'],
            '经营范围': data['businessRange'],
            '单位网址': data['companyUrl'],
            '法人': data['legalPerson'],
            '主营业务': data['mainBusiness'],
            '注册地址': data['registeredAddress'],
            '证书网址': data['companyCommitment'],
        }
        print(dit)
        csv_writer.writerow(dit)