免责声明:本文仅做分享参考~
目录
CrawlSpider
介绍
CrawlSpider类:定义了一些规则来做数据爬取,从爬取的网页中获取链接并进行继续爬取.
python
创建方式:scrapy genspider -t crawl 爬虫名 爬虫域名
scrapy genspider -t crawl zz zz.com
(子类就是牛,因为可以继承啊,还可以有自己的方法.)
spider类--》ZzscSpider类
spider类--》CrawLSpider类--》XjSpider类
CrawLSpider类:定义了一些规则来做数据爬取,从爬取的网页中获取链接并进行继续爬取
这个子类注意好 规则 , 有助于帮你跟进追踪类似数据(多页,类似html结构), 是这个子类最大的特点.
--正则匹配 !!!
新疆xxedu
xj.py
python
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
class XjSpider(CrawlSpider):
name = "xj"
# allowed_domains = ["xj.com"]
start_urls = ["https://www.xjie.edu.cn/tzgg1.htm"]
# 规则,根据正则匹配: \ 转义, ^ 开始, $ 结束, () 分组, [] 范围, | 或, + 1次或多次, * 0次或多次
# 获取详情页链接:
# rules = (Rule(LinkExtractor(allow=r"Items/"), callback="parse_item", follow=True),)
# follow属性为True,会自动跟进详情页链接,并调用parse_item方法处理.
rules = (
Rule(
LinkExtractor(allow=r"info/1061/.*?\.htm"),
callback="parse_item",
follow=False,
),
Rule(LinkExtractor(allow=r"tzgg1/.*?\.htm"), follow=True),
)
count = 0
# 匹配100次,就进入parse_item方法100次.
def parse_item(self, response):
# print(response.request.headers) #查看伪装的请求头
self.count += 1
# 基于详情页,获取标题:
title = response.xpath("//h3/text()").get()
print(title, self.count) # 竟然也自动拿到了分页的url.
item = {}
# item["domain_id"] = response.xpath('//input[@id="sid"]/@value').get()
# item["name"] = response.xpath('//div[@id="name"]').get()
# item["description"] = response.xpath('//div[@id="description"]').get()
return item
中间件
scrapy中有两个中间件:
下载中间件:位于引擎和下载器中间.
爬虫中间件:位于引擎和爬虫中间.(一般不用)
下载中间件的作用:
用来篡改请求和响应,比如篡改请求:加一些请求头,加代理等等;
篡改响应就是更改响应的内容.
下载器
拦截请求 --伪装
拦截响应 --修改返回内容
部分middlewares.py
settings.py里面的是全局设置,在这你可以自己定义爬虫的伪装~~
python
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
# 引擎把请求对象交给下载器,下载器把响应对象交给引擎.
class ScrapyDemo1SpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn't have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info("Spider opened: %s" % spider.name)
class ScrapyDemo1DownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
# 发起请求执行的方法
def process_request(self, request, spider):
# Called for each request that goes through the downloader
# middleware.
# Must either:
# - return None: continue processing this request
# - or return a Response object
# - or return a Request object
# - or raise IgnoreRequest: process_exception() methods of
# installed downloader middleware will be called
#
# if spider.name == "wyxw": # 判断,如果是指定爬虫,则修改请求头.
request.headers["User-Agent"] = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
)
# cookie伪装
# request.cookies["name"] = "value"
# 如果更换ip地址,使用代理:
# request.meta['proxy'] = 'https://ip:端口'
print(request)
return None
# 得到响应执行的方法
#
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
# 处理异常执行的方法
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info("Spider opened: %s" % spider.name)
为什么要用中间件的原因:
网页是需要滚动加载数据的,所以借助自动化工具去拿到所有的页面数据,利用scrapy中 间件整合自动化工具.
网易新闻爬取四个板块数据
国内 国际 军事 航空 因为它们的页面结构一致,取值方式一致.
四个版块的数据 通过访问url 只能拿到50条数据 其中有一个版块需要手动点击 加载更多才可以继续把数据加载完整 其它的几个版块几乎都是一直滚动滚动条,直到数据完全加载为止
解决问题?--> 滚动条的操作 requests scrapy也没有,只有自动化技术才可以实现滚动滚动条 怎么样才可以scrapy结合自动化技术呢?
使用自动化技术 加载所有数据 !
wyxw.py
爬虫文件
python
import scrapy
"""
新闻页面的数据是通过滚动翻页加载
如果要获取所有的数据:
解决办法:
传统的方式:通过网络面板看分页加载的请求是什么
自动化工具的方式:让自动化工具实现鼠标滚动
"""
class WyxwSpider(scrapy.Spider):
name = "wyxw"
# allowed_domains = ["wyxw.com"]
start_urls = ["https://news.163.com/"]
count = 0
def parse(self, response):
# 解析四个版块的url
lis = response.xpath('//div[@class="ns_area list"]/ul/li')
# 获取四个版块的li
# lis[1:3] lis[1] lis[2]
# lis[4:6] lis[4] lis[5]
target_li = lis[1:3] + lis[4:6]
# print(target_li)
# 循环li 拿到每一个,li的a标签的hraf属性值 再发起请求
for li in target_li:
href = li.xpath("./a/@href").get()
# 发起请求
yield scrapy.Request(url=href, callback=self.news_parse)
# 解析每个版块的数据
# HtmlResponse(元素) 还是 scrapy的response?是自己返回的HtmlResponse,所以要以元素面板为主
# 因为数据是通过自动化工具获取到的,自动化获取的数据都是元素面板数据
def news_parse(self, response):
divs = response.xpath('//div[@class="ndi_main"]/div') # 从元素面板看的
# divs = response.xpath('//div[@class="hidden"]/div') # 从响应内容看的
for div in divs:
self.count += 1
title = div.xpath(".//h3/a/text()").get() # 从元素面板看的
# title = div.xpath('./a/text()').get() # 从响应内容看的
print(title, self.count)
完整的middlewares.py
process_request 函数里面 伪装请求.
process_response 函数里面 拦截请求,返回我们想要的数据.
python
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import scrapy
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
from DrissionPage._pages.chromium_page import ChromiumPage
from scrapy.http import HtmlResponse
class Scrapy4SpiderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the spider middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
def process_spider_input(self, response, spider):
# Called for each response that goes through the spider
# middleware and into the spider.
# Should return None or raise an exception.
return None
def process_spider_output(self, response, result, spider):
# Called with the results returned from the Spider, after
# it has processed the response.
# Must return an iterable of Request, or item objects.
for i in result:
yield i
def process_spider_exception(self, response, exception, spider):
# Called when a spider or process_spider_input() method
# (from other spider middleware) raises an exception.
# Should return either None or an iterable of Request or item objects.
pass
def process_start_requests(self, start_requests, spider):
# Called with the start requests of the spider, and works
# similarly to the process_spider_output() method, except
# that it doesn't have a response associated.
# Must return only requests (not items).
for r in start_requests:
yield r
def spider_opened(self, spider):
spider.logger.info("Spider opened: %s" % spider.name)
class Scrapy4DownloaderMiddleware:
# Not all methods need to be defined. If a method is not defined,
# scrapy acts as if the downloader middleware does not modify the
# passed objects.
@classmethod
def from_crawler(cls, crawler):
# This method is used by Scrapy to create your spiders.
s = cls()
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
return s
# 发起请求执行的方法
def process_request(self, request, spider):
c_dict = {'cookiesu': '291715258858774', ' device_id': '2bad6b34106f8be1d2762204c306aa5b',
' smidV2': '20240509204738adc267d3b66fca8adf0d37b8ac9a1e8800dbcffc52451ef70', ' s': 'ak145lfb3s',
' __utma': '1.1012417897.1720177183.1720177183.1720435739.2',
' __utmz': '1.1720435739.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic',
' acw_tc': '2760827c17230343015564537e9504eebc062a87e8d3ba4425778641e1164e',
' xq_a_token': 'fb0f503ef881090db449e976c330f1f2d626c371', ' xqat': 'fb0f503ef881090db449e976c330f1f2d626c371',
' xq_r_token': '967c806e113fbcb1e314d5ef2dc20f1dd8e66be3',
' xq_id_token': 'eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOi0xLCJpc3MiOiJ1YyIsImV4cCI6MTcyNTMyNDc3NSwiY3RtIjoxNzIzMDM0MzAxODI4LCJjaWQiOiJkOWQwbjRBWnVwIn0.I0pylYRqjdDOE0QbAuLFr5rUCFG1Sj13kTEHAdap2fEse0i12LG7a-14rhvQKsK9G0F7VZSWDYBsI82mqHimsBbWvgefkOy-g8V_nhb-ntGBubbVHlfjv7y-3GDatzcZPfgYvu7m0wEu77PcWdKqJR-KwZsZjQVwaKiHcFuvFUmpfYN942D6YY2MIzgWJxSQCX_t4f1YujRJRsKLvrq9QbVIqvJu-SpPT1SJfSPT9e7h_ERkU0QOsmgARJVfivkAAFM2cyb7HKJsHQSqVSU6hcIq4CMs5r90IsrZ4fOL5cUqaGNT58qkjx-flta27QhCIeHxexi1K95TKZTcD8EygA',
' u': '291715258858774',
' Hm_lvt_1db88642e346389874251b5a1eded6e3': '1722350318,1722863131,1722925588,1723034326',
' Hm_lpvt_1db88642e346389874251b5a1eded6e3': '1723034326', ' HMACCOUNT': '13108745FF137EDD',
' .thumbcache_f24b8bbe5a5934237bbc0eda20c1b6e7': 'YEsBB9JFA5Q4gQHJIe1Lx6JjvpZzcuUljYTfjFKm3lmCSpRZMpoNmnSBV0UptK3ripTe4xifyqRUZO/LEPx6Iw%3D%3D',
' ssxmod_itna': 'WqIx9DgD0jD==0dGQDHWWoeeqBKorhBoC7ik8qGN6xYDZDiqAPGhDC4bUxD5poPoqWW3pi4kiYr23PZF2E2GaeBm8EXTDU4i8DCwiK=ODem=D5xGoDPxDeDAQKiTDY4DdjpNv=DEDeKDRDAQDzLdyDGfBDYP9QqDgSqDBGOdDKqGgzTxD0TxNaiqq8GKKvkd5qjbDAwGgniq9D0UdxBLxAax9+j9kaBUg8ZaPT2jx5eGuDG6DOqGmSfb3zdNPvAhWmY4sm75Ymbq4n75e8YervGPrPuDNh0wKY7DoBGp=GDLMxDfT0bD',
' ssxmod_itna2': 'WqIx9DgD0jD==0dGQDHWWoeeqBKorhBoC7ik4A=W=e4D/D0hq7P7phOF4WG2WCxjKD2WYD=='}
if spider.name == 'xueqiu':
# ua Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0
request.headers['referer'] = 'https://xueqiu.com/'
request.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0'
# 失效 request.headers['cookie'] = 'cookiesu=291715258858774; device_id=2bad6b34106f8be1d2762204c306aa5b; smidV2=20240509204738adc267d3b66fca8adf0d37b8ac9a1e8800dbcffc52451ef70; s=ak145lfb3s; __utma=1.1012417897.1720177183.1720177183.1720435739.2; __utmz=1.1720435739.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; acw_tc=276077b217228631300332345e2956486032bd992e9f60b5689d9b248c1491; xq_a_token=fb0f503ef881090db449e976c330f1f2d626c371; xq_r_token=967c806e113fbcb1e314d5ef2dc20f1dd8e66be3; xq_id_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOi0xLCJpc3MiOiJ1YyIsImV4cCI6MTcyNTMyNDc3NSwiY3RtIjoxNzIyODYzMDg5MDIwLCJjaWQiOiJkOWQwbjRBWnVwIn0.AdH2II1N2kGggG7ZgmP8MOgNPdxMoewCSK-gyWYBkw7zFExn6gaYV6YU8ReNkp2F5CBohxjZYyVyLtn98MJfk9dDwIe8ypTgXLkI_a5R1O1og5Fy6BeFv6FUJqgp8EVt8EvHBOYfRNl9iGtgrO3V_R0fJXq1aJTpV8lopNwEAzQbHRK58uXcbaoOwkUcX8MOv6XR-eGqnHYRSJ35P769atb6vF05LqutQphcairWpGGgWJc9fMhVBym_GkOxy4_AWaURWf8Zpge7dJQszkCo-ljPbBP94vz3zM_PTnussZV3jeTRmacaJcHTee6mlE00hrtrAFZNf7UIjnpqbdzvjw; u=291715258858774; Hm_lvt_1db88642e346389874251b5a1eded6e3=1720435739,1722235054,1722350318,1722863131; HMACCOUNT=13108745FF137EDD; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1722863143; .thumbcache_f24b8bbe5a5934237bbc0eda20c1b6e7=ha9Q037kb9An+HmEECE3+OwwxbDqAMnKP2QzosxWbRaWA89HDsKTwe/0XwLZXhS9S5OvxrWGFu6LMFW2iVI9xw%3D%3D; ssxmod_itna=QqmxgD0Cq4ciDXYDHiuYYK0=e4D5Dk3DA2LOni44qGNjKYDZDiqAPGhDC4fUerbrq5=7oC03qeiB4hvpkKrmDKijF/qqeDHxY=DUa7aeDxpq0rD74irDDxD3wxneD+D04kguOqi3DhxGQD3qGylR=DA3tDbh=uDiU/DDUOB4G2D7UyiDDli0TeA8mk7CtUCQ03xGUxIqqQ7qDMUeGX87Fe7db86PhMIwaHamPuKCiDtqD94m=DbRL3vB6lWW+r7hGrGBvNGGYQKDqNQ7eeKDmjvfhu2GDile4tKOG5nBpan/zeDDfj0bD===; ssxmod_itna2=QqmxgD0Cq4ciDXYDHiuYYK0=e4D5Dk3DA2LOni4A=c==D/QKDFxnAO9aP7QmHGcDYK4xD==='
# request.cookies['键'] = '值'
# request.cookies['cookiesu'] = '291715258858774'
for key,value in c_dict.items():
request.cookies[key] = value
# print(request)
# 如果更换ip地址,使用代理
# request.meta['proxy'] = 'https://ip:端口'
return None
# 得到响应执行的方法
# 结合自动化工具
def process_response(self, request, response, spider):
# url必须是四个版块的其中一个
if spider.name == 'wyxw':
url = request.url
# url = 'https://news.163.com/domestic/'
dp = ChromiumPage()
dp.get(url)
# 滚动到页面底部 一次性不能完成
while True:
is_block = dp.ele('.load_more_tip').attr('style')
if is_block == 'display: block;':
# 已经到最后了
break
dp.scroll.to_bottom()
# 国内页面需要额外手动点击
try:
# 只有一个页面能够点击加载更多,加入异常处理,如果出现了异常,进行下一次循环
click_tag = dp.ele('text:加载更多')
click_tag.click()
except:
continue
# 创建响应对象,把自动化工具得到的html数据放入到对象中
# 返回给引擎,由引擎交给spider
# HtmlResponse响应对象
'''
class HtmlResponse:
def __init__(self,request,url,body):
HtmlResponse(url=url,request=request,body=dp.html)
'''
return HtmlResponse(url=url,request=request,body=dp.html,encoding='utf-8')
return response
# 遇到异常执行的方法
def process_exception(self, request, exception, spider):
# Called when a download handler or a process_request()
# (from other downloader middleware) raises an exception.
# Must either:
# - return None: continue processing this exception
# - return a Response object: stops process_exception() chain
# - return a Request object: stops process_exception() chain
pass
def spider_opened(self, spider):
spider.logger.info("Spider opened: %s" % spider.name)