爬虫实战-房天下(bengbu.zu.fang.com/)数据爬取

详细代码链接https://flowus.cn/hbzx/3c42674d-8e6f-42e3-a3f6-bc1258034676

import requests

from lxml import etree #xpath解析库

def 源代码(url):

cookies = {

'global_cookie': 'xeqnmumh38dvpj96uzseftwdr20lvkwkfb9',

'otherid': 'b44a1837638234f1a0a15e37877e0685',

'g_sourcepage': 'zf_fy%5Elb_pc',

'__utma': '147393320.818863681.1714391725.1714391725.1714391725.1',

'__utmc': '147393320',

'__utmz': '147393320.1714391725.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none)',

'__utmt_t0': '1',

'__utmt_t1': '1',

'__utmt_t2': '1',

'keyWord_recenthousebengbu': '%5b%7b%22name%22%3a%22%e9%be%99%e5%ad%90%e6%b9%96%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a011914%2fs31%2f%22%2c%22sort%22%3a1%7d%5d',

'city': 'sh',

'ASP.NET_SessionId': '4fpr5u3w5zqqzitrnwafk3cr',

'zf_csrfcookie': '1cCMHJcsaY7XgtGVMdiMdsydBeGKPxx7G1pYnsK0yn4vRI361O_aeBQfC7SAKi4gktL0kQ2',

'unique_cookie': 'U_xeqnmumh38dvpj96uzseftwdr20lvkwkfb9*6',

'__utmb': '147393320.18.10.1714391725',

}

headers = {

'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',

'accept-language': 'zh-CN,zh;q=0.9',

'cache-control': 'max-age=0',

'cookie': 'global_cookie=xeqnmumh38dvpj96uzseftwdr20lvkwkfb9; otherid=b44a1837638234f1a0a15e37877e0685; g_sourcepage=zf_fy%5Elb_pc; __utma=147393320.818863681.1714391725.1714391725.1714391725.1; __utmc=147393320; __utmz=147393320.1714391725.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt_t0=1; __utmt_t1=1; __utmt_t2=1; keyWord_recenthousebengbu=%5b%7b%22name%22%3a%22%e9%be%99%e5%ad%90%e6%b9%96%22%2c%22detailName%22%3a%22%22%2c%22url%22%3a%22%2fhouse-a011914%2fs31%2f%22%2c%22sort%22%3a1%7d%5d; city=sh; ASP.NET_SessionId=4fpr5u3w5zqqzitrnwafk3cr; zf_csrfcookie=1cCMHJcsaY7XgtGVMdiMdsydBeGKPxx7G1pYnsK0yn4vRI361O_aeBQfC7SAKi4gktL0kQ2; unique_cookie=U_xeqnmumh38dvpj96uzseftwdr20lvkwkfb9*6; __utmb=147393320.18.10.1714391725',

'priority': 'u=0, i',

'referer': 'https://sh.zu.fang.com/house/i33/',

'sec-ch-ua': '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',

'sec-ch-ua-mobile': '?0',

'sec-ch-ua-platform': '"Windows"',

'sec-fetch-dest': 'document',

'sec-fetch-mode': 'navigate',

'sec-fetch-site': 'same-origin',

'sec-fetch-user': '?1',

'upgrade-insecure-requests': '1',

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',

}

response = requests.get(url, cookies=cookies, headers=headers).text

#response是源代码

return response

if name=='main':

for fan in range(1,10):

url='https://sh.zu.fang.com/house/i3{fan}/'

res=源代码(url)

res=etree.HTML(res) #初始化

#//*[@id="rentid_D09_01_02"]/a a后面没有加/text()

#//*[@id="rentid_D09_60_02"]/a

#//*[@id="rentid_D09_02_02"]/a

#//*[@id="rentid_D09_31_02"]/a

for i in range(1,61):

if i<10:

ix='0'+str(i)

xp=f'//*[@id="rentid_D09_{ix}_02"]/a/text()'

else:

xp=f'//*[@id="rentid_D09_{i}_02"]/a/text()'

title=res.xpath(xp)[0]

print(title,end=' ')

xp=f'//*[@id="listBox"]/div[3]/dl[{i}]/dd/p[2]/text()'

#//*[@id="listBox"]/div[3]/dl[60]/dd/p[2]

p=res.xpath(xp)[0].strip()

print(p,end=' ')

xp=f'//*[@id="listBox"]/div[3]/dl[{i}]/dd/div[2]/p/span/text()'

#//*[@id="listBox"]/div[3]/dl[60]/dd/div[2]/p/span

jg=res.xpath(xp)[0]

print(jg)

相关推荐
华科云商xiao徐2 小时前
告别IP被封!分布式爬虫的“隐身”与“分身”术
爬虫·数据挖掘·数据分析
q567315236 小时前
告别低效:构建健壮R爬虫的工程思维
开发语言·爬虫·r语言
一个天蝎座 白勺 程序猿15 小时前
Python爬虫(47)Python异步爬虫与K8S弹性伸缩:构建百万级并发数据采集引擎
爬虫·python·kubernetes
华科云商xiao徐1 天前
告别低效:构建健壮R爬虫的工程思维
爬虫
熊猫钓鱼>_>2 天前
2025反爬虫之战札记:从robots.txt到多层防御的攻防进化史
开发语言·c++·爬虫
Blurpath2 天前
如何利用静态代理IP优化爬虫策略?从基础到实战的完整指南
爬虫·网络协议·ip代理·住宅代理
wjayg2252 天前
网络爬虫是自动从互联网上采集数据的程序
爬虫
IT毕设实战小研2 天前
2026届大数据毕业设计选题推荐-基于大数据旅游数据分析与推荐系统 爬虫数据可视化分析
大数据·人工智能·爬虫·机器学习·架构·数据分析·课程设计
Villiam_AY2 天前
使用 chromedp 高效爬取 Bing 搜索结果
后端·爬虫·golang
爬虫程序猿3 天前
利用 Java 爬虫获取淘宝商品 SKU 详细信息实战指南
java·开发语言·爬虫