"揭秘CentosChina爬虫项目：掌握Scrapy框架的必备技巧与数据库设计"

Centoschina

项目要求

爬取centoschina_cn的所有问题，包括文章标题和内容

数据库表设计

库表设计：

数据展示：

项目亮点

• 低耦合，高内聚。

  爬虫专有settings

  custom_settings = custom_settings_for_centoschina_cn

  custom_settings_for_centoschina_cn = {
      'MYSQL_USER': 'root',
      'MYSQL_PWD': '123456',
      'MYSQL_DB': 'questions',
  }

• DownloaderMiddleware使用

  class CentoschinaDownloaderMiddleware:
      # Not all methods need to be defined. If a method is not defined,
      # scrapy acts as if the downloader middleware does not modify the
      # passed objects.
  
      @classmethod
      def from_crawler(cls, crawler):
          # This method is used by Scrapy to create your spiders.
          s = cls()
          crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
          return s
  
      # 处理请求
      def process_request(self, request, spider):
          # Called for each request that goes through the downloader
          # middleware.
  
          # Must either:
          # - return None: continue processing this request 继续执行下一步操作，不处理默认返回None
          # - or return a Response object 直接返回响应, 如scrapy和pyppeteer不需要用下载器中间件访问外网，直接返回响应, pyppeteer有插件，一般和scrapy还能配合，selenium不行，没有插件
          # - or return a Request object 将请求返回到schdular的调度队列中供以后重新访问
          # - or raise IgnoreRequest: process_exception() methods of
          #   installed downloader middleware will be called
          return None
  
      # 处理响应
      def process_response(self, request, response, spider):
          # Called with the response returned from the downloader.
  
          # Must either;
          # - return a Response object 返回响应结果
          # - return a Request object 结果不对（判断结果对不对一般判断状态码和内容大小）一般返回request，也是将请求返回到schdular的调度队列中供以后重新访问
          # - or raise IgnoreRequest
          return response
  
      # 处理异常：如超时错误等
      def process_exception(self, request, exception, spider):
          # Called when a download handler or a process_request()
          # (from other downloader middleware) raises an exception.
  
          # Must either:
          # - return None: continue processing this exception 继续执行下一步，没有异常
          # - return a Response object: stops process_exception() chain 如果其返回一个 Response 对象，则已安装的中间件链的 process_response() 方法被调用。Scrapy将不会调用任何其他中间件的 process_exception() 方法。
          # - return a Request object: stops process_exception() chain 将请求返回到schdular的调度队列中供以后重新访问
          pass
  
      def spider_opened(self, spider):
          spider.logger.info("Spider opened: %s" % spider.name)

• DownloaderMiddleware中抛弃请求写法

  • 适用场景：请求异常，换代理或者换cookie等操作

  # from scrapy.exceptions import IgnoreRequest
  # raise IgnoreRequest(f'Failed to retrieve {request.url} after {max_retries} retries')

  例子：处理下载异常并重试请求

  import logging
  from scrapy.exceptions import IgnoreRequest
  
  class RetryExceptionMiddleware:
      def __init__(self):
          self.logger = logging.getLogger(__name__)
  
      def process_exception(self, request, exception, spider):
          # 记录异常信息
          self.logger.warning(f'Exception {exception} occurred while processing {request.url}')
          
          # 检查是否达到重试次数限制
          max_retries = 3
          retries = request.meta.get('retry_times', 0) + 1
          
          if retries <= max_retries:
              self.logger.info(f'Retrying {request.url} (retry {retries}/{max_retries})')
              # 增加重试次数
              request.meta['retry_times'] = retries
              return request
          else:
              self.logger.error(f'Failed to retrieve {request.url} after {max_retries} retries')
              raise IgnoreRequest(f'Failed to retrieve {request.url} after {max_retries} retries')

  例子：切换代理

  import random
  
  class SwitchProxyMiddleware:
      def __init__(self, proxy_list):
          self.proxy_list = proxy_list
          self.logger = logging.getLogger(__name__)
  
      @classmethod
      def from_crawler(cls, crawler):
          proxy_list = crawler.settings.get('PROXY_LIST')
          return cls(proxy_list)
  
      def process_exception(self, request, exception, spider):
          self.logger.warning(f'Exception {exception} occurred while processing {request.url}')
          
          # 切换代理
          proxy = random.choice(self.proxy_list)
          self.logger.info(f'Switching proxy to {proxy}')
          request.meta['proxy'] = proxy
          
          # 重试请求
          return request

• piplines中抛弃item写法

  • 适用场景：数据清洗、去重、验证等操作

  # from scrapy.exceptions import DropItem
  # raise DropItem("Duplicate item found: %s" % item)

• 保存到文件(通过命令)

  from scrapy.cmdline import execute
  execute(['scrapy', 'crawl', 'centoschina_cn', '-o', 'questions.csv'])

更多精致内容: