16爬虫：使用requests和scrapy分别从链家获取二手房信息

requests

import requests
import parsel
import time
from random import randint

# 写成方法的形式，方便翻页抓取
def spider(url, headers): # 爬虫方法
    response = requests.get(url=url, headers=headers)
    return response

def parse_response(response):# 解析响应
    response.encoding = "utf-8"
    selector = parsel.Selector(response.text)
    infos = selector.xpath('//div[@class="info clear"]')
    for info in infos:
        title = info.xpath('.//div[@class="title"]/a/text()').get()
        position = info.xpath('.//div[@class="flood"]//a[2]/text()').get()
        houseInfo = info.xpath('.//div[@class="address"]/div/text()').get()
        five = info.xpath('.//div[@class="tag"]/span[@class="five"]/text()').get()
        if five == None:
            five = '未知，需要联系户主或者中介'
        haskey = info.xpath('.//div[@class="tag"]/span[@class="haskey"]/text()').get()
        if haskey == None:
            haskey = '未知，需要联系户主或者中介'
        totalPrice = info.xpath('.//div[@class="priceInfo"]//span/text()').get() + '万'
        print('===================================================================')
        print('房子的形容词：' + title)
        print('房子的坐落位置：' + position)
        print('房子的基本信息：' + houseInfo)
        print('房本是否满五年：' + five)
        print('是否能够随时看房：' + haskey)
        print('房屋的总售价：' + totalPrice)

if __name__ == '__main__':
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
    }
    for i in range(1,30): # 实际有100页
        time.sleep(randint(1,5))
        url = "https://yt.lianjia.com/ershoufang/pg{}".format(i)
        response = spider(url, headers)
        parse_response(response)

scrapy

runner.py

from scrapy.cmdline import execute
# 在中间件中完成增加请求头
if __name__ == '__main__':
    execute("scrapy crawl secondHandHouse".split())

spider

import scrapy
from lianjia.items import LianjiaItem

class SecondhandhouseSpider(scrapy.Spider):
    name = "secondHandHouse"
    allowed_domains = ["lianjia.com"]
    # 假设我们爬取前30也的内容
    start_urls = []
    for i in range(1,5):
        url = "https://yt.lianjia.com/ershoufang/pg{}".format(i)
        start_urls.append(url) # 构建一个完整的url列表，里面存储我们所有需要爬取的url网址


    def parse(self, response,**kwargs):
        print(response.status)
        infos = response.xpath('//div[@class="info clear"]')
        for info in infos:
            title = info.xpath('.//div[@class="title"]/a/text()').extract_first()
            position = info.xpath('.//div[@class="flood"]//a[2]/text()').extract_first()
            houseInfo = info.xpath('.//div[@class="address"]/div/text()').extract_first()
            five = info.xpath('.//div[@class="tag"]/span[@class="five"]/text()').extract_first()
            if five == None:
                five = '未知，需要联系户主或者中介'
            haskey = info.xpath('.//div[@class="tag"]/span[@class="haskey"]/text()').extract_first()
            if haskey == None:
                haskey = '未知，需要联系户主或者中介'
            totalPrice = info.xpath('.//div[@class="priceInfo"]//span/text()').extract_first() + '万'
            print('===================================================================')
            print('房子的形容词：' + title)
            print('房子的坐落位置：' + position)
            print('房子的基本信息：' + houseInfo)
            print('房本是否满五年：' + five)
            print('是否能够随时看房：' + haskey)
            print('房屋的总售价：' + totalPrice)
        #     item = LianjiaItem()
        #     item['title'] = title
        #     item['position'] = position
        #     item['houseInfo'] = houseInfo
        #     item['five'] = five
        #     item['totalPrice'] = totalPrice
        #     yield item

settings.py

# Scrapy settings for lianjia project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = "lianjia"

SPIDER_MODULES = ["lianjia.spiders"]
NEWSPIDER_MODULE = "lianjia.spiders"


# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
LOG_LEVEL = "WARNING"

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
#    "Accept-Language": "en",
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    "lianjia.middlewares.LianjiaSpiderMiddleware": 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    "lianjia.middlewares.LianjiaDownloaderMiddleware": 543,
# }

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    "scrapy.extensions.telnet.TelnetConsole": None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   "lianjia.pipelines.LianjiaPipeline": 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = "httpcache"
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"

# Set settings whose default value is deprecated to a future-proof value
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"

在scrapy中只修改了上述的内容，其他的组件代码保持不动。

结果