【python】数据爬虫，抓取并分析豆瓣电影信息

ClearStari992024-09-23 8:32

url-lib-request模块获取HTML

spider.py

网页下载，链接提取，数据存储

http请求：

网址url，

http-header: (1)用户代理；(2)浏览器版本

网页下载用 urllib库。

python 复制代码 
# 将豆瓣电影top250的第一页下载

import urllib.request # 导入url-lib-request模块

# 设置网址
url = "https://movie.douban.com/top250"

# http头信息
header = {
    "User-Agent":"Mozilla/5.0(Windows NT 10.0;WOW64)"
    "AppleWebKit/537.36(KHTML, like Gecko)"
    "Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.8131"
}

# 构造一个Request请求对象
req = urllib.request.Request(url = url, headers = header)

# 向豆瓣服务器发送请求，获取到回复response
response = urllib.request.urlopen(req)

# 读取回复并使用UTF-8编码进行解码
html = response.read().decode("utf-8")

# 打印HTML
print(html)

打印满足条件的网址

python 复制代码 
import urllib.request # 导入url-lib-request模块

def download_html(url):
    # http头信息
    header = {
        "User-Agent":"Mozilla/5.0(Windows NT 10.0; WOW64)"
        "AppleWebKit/537.36(KHTML, like Gecko)"
        "Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.8131"
    }
    # 构造一个Request请求对象
    req = urllib.request.Request(url = url, headers = header)
    # 向豆瓣服务器发送请求，获取到回复response
    response = urllib.request.urlopen(req)
    # 读取回复并使用UTF-8编码进行解码
    html = response.read().decode("utf-8")

    return html

# 调用函数download_html，下载豆瓣top250第一页，保存至html
html = download_html("https://movie.douban.com/top250")

import re # 导入re正则库

# 电影介绍页网址的正则表达式
pattern = 'https://movie.douban.com/subject/[0-9]+/'

# 从html中提取全部满足电影介绍页的网址pattern的url
urls = re.findall(pattern, html)

urls = set(urls) # 通过set集合去重

# 打印出提取出的网址数量和具体的网址
print("urls count = %d" % (len(urls)))
for url in urls:
    print(url)

douban.txt 文件内容：

txt 复制代码 
https://movie.douban.com/top250
https://movie.douban.com/top250?start=25
https://movie.douban.com/top250?start=50
https://movie.douban.com/top250?start=75
https://movie.douban.com/top250?start=100
https://movie.douban.com/top250?start=125
https://movie.douban.com/top250?start=150
https://movie.douban.com/top250?start=175
https://movie.douban.com/top250?start=200
https://movie.douban.com/top250?start=225

写入到文件中

python 复制代码 
import urllib.request # 导入url-lib-request模块
import re # 导入re正则库

def download_html(url):
    # http头信息
    header = {
        "User-Agent":"Mozilla/5.0(Windows NT 10.0; WOW64)"
        "AppleWebKit/537.36(KHTML, like Gecko)"
        "Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.8131"
    }
    # 构造一个Request请求对象
    req = urllib.request.Request(url = url, headers = header)
    # 向豆瓣服务器发送请求，获取到回复response
    response = urllib.request.urlopen(req)
    # 读取回复并使用UTF-8编码进行解码
    html = response.read().decode("utf-8")

    return html

# 调用函数download_html，下载豆瓣top250第一页，保存至html
html = download_html("https://movie.douban.com/top250")

# 电影介绍页网址提取的功能封装至函数extract_url中，
# 函数输入html，返回html中提取的url连接
def extract_url(html):
    # 电影介绍页网址的正则表达式
    pattern = 'https://movie.douban.com/subject/[0-9]+/'
    # 从html中提取全部满足电影介绍页的网址pattern的url
    urls = re.findall(pattern, html)
    return set(urls) # 通过set集合去重

# 以读取方式打开豆瓣电影列表文件
file = open('douban.txt', 'r')
# 然后以写模式打开movie.txt，用来存储提取出的电影介绍页网址
output = open('movie.txt', 'w')

# 将douban中的网址按行读入
lines = file.readlines()

for url in lines:
    # 遍历每一行url，通过strip函数去掉首尾空字符
    url = url.strip()
    print(url) # 打印url作为调试信息
    # 调用download_html获取url对应的html代码
    html = download_html(url)
    # 调用extract_url提取html中的电影介绍页链接网址
    urls = extract_url(html)
    for url in urls: # 遍历url集合
        output.write(url + '\n') # 将每个url都写入movie文件
# 最后关闭douban和movie两个文件
file.close()
output.close()

movie.txt 结果：

txt 复制代码 
https://movie.douban.com/subject/1889243/
https://movie.douban.com/subject/1849031/
https://movie.douban.com/subject/1291561/
https://movie.douban.com/subject/6786002/
https://movie.douban.com/subject/20495023/
https://movie.douban.com/subject/1291841/
https://movie.douban.com/subject/3011091/
https://movie.douban.com/subject/1307914/
https://movie.douban.com/subject/1291549/
https://movie.douban.com/subject/1292213/
https://movie.douban.com/subject/1295124/
https://movie.douban.com/subject/1292063/
https://movie.douban.com/subject/2131459/
https://movie.douban.com/subject/1292052/
https://movie.douban.com/subject/1291546/
https://movie.douban.com/subject/1292064/
https://movie.douban.com/subject/25662329/
https://movie.douban.com/subject/1292001/
https://movie.douban.com/subject/3793023/
https://movie.douban.com/subject/1292720/
https://movie.douban.com/subject/1295644/
https://movie.douban.com/subject/5912992/
https://movie.douban.com/subject/1296141/
https://movie.douban.com/subject/3541415/
https://movie.douban.com/subject/1292722/
https://movie.douban.com/subject/1292365/
https://movie.douban.com/subject/1305487/
https://movie.douban.com/subject/1293182/
https://movie.douban.com/subject/21937452/
https://movie.douban.com/subject/1291560/
https://movie.douban.com/subject/1291552/
https://movie.douban.com/subject/1296736/
https://movie.douban.com/subject/3742360/
https://movie.douban.com/subject/1291572/
https://movie.douban.com/subject/1295038/
https://movie.douban.com/subject/1293172/
https://movie.douban.com/subject/1299398/
https://movie.douban.com/subject/30170448/
https://movie.douban.com/subject/1291858/
https://movie.douban.com/subject/26387939/
https://movie.douban.com/subject/1300267/
https://movie.douban.com/subject/3319755/
https://movie.douban.com/subject/26752088/
https://movie.douban.com/subject/2129039/
https://movie.douban.com/subject/1298624/
https://movie.douban.com/subject/1851857/
https://movie.douban.com/subject/1308807/
https://movie.douban.com/subject/1929463/
https://movie.douban.com/subject/25958717/
https://movie.douban.com/subject/1291583/
https://movie.douban.com/subject/1291548/
https://movie.douban.com/subject/1293839/
https://movie.douban.com/subject/27060077/
https://movie.douban.com/subject/3011235/
https://movie.douban.com/subject/1301753/
https://movie.douban.com/subject/21937445/
https://movie.douban.com/subject/1652587/
https://movie.douban.com/subject/1292849/
https://movie.douban.com/subject/1418019/
https://movie.douban.com/subject/1299131/
https://movie.douban.com/subject/26580232/
https://movie.douban.com/subject/1292402/
https://movie.douban.com/subject/1292220/
https://movie.douban.com/subject/1485260/
https://movie.douban.com/subject/1291818/
https://movie.douban.com/subject/1293350/
https://movie.douban.com/subject/1291543/
https://movie.douban.com/subject/3008247/
https://movie.douban.com/subject/1291571/
https://movie.douban.com/subject/1291828/
https://movie.douban.com/subject/1292000/
https://movie.douban.com/subject/1306029/
https://movie.douban.com/subject/1900841/
https://movie.douban.com/subject/1294408/
https://movie.douban.com/subject/1291843/
https://movie.douban.com/subject/1298070/
https://movie.douban.com/subject/1306249/
https://movie.douban.com/subject/11525673/
https://movie.douban.com/subject/1302425/
https://movie.douban.com/subject/1292343/
https://movie.douban.com/subject/6985810/
https://movie.douban.com/subject/1292224/
https://movie.douban.com/subject/1292434/
https://movie.douban.com/subject/1291544/
https://movie.douban.com/subject/2334904/
https://movie.douban.com/subject/3442220/
https://movie.douban.com/subject/1300299/
https://movie.douban.com/subject/1780330/
https://movie.douban.com/subject/1292656/
https://movie.douban.com/subject/5322596/
https://movie.douban.com/subject/1296996/
https://movie.douban.com/subject/1303021/
https://movie.douban.com/subject/1292223/
https://movie.douban.com/subject/1291832/
https://movie.douban.com/subject/1293544/
https://movie.douban.com/subject/1294371/
https://movie.douban.com/subject/1297192/
https://movie.douban.com/subject/1865703/
https://movie.douban.com/subject/1292262/
https://movie.douban.com/subject/1292679/
https://movie.douban.com/subject/3395373/
https://movie.douban.com/subject/1418834/
https://movie.douban.com/subject/1305164/
https://movie.douban.com/subject/1294639/
https://movie.douban.com/subject/21318488/
https://movie.douban.com/subject/10777687/
https://movie.douban.com/subject/2149806/
https://movie.douban.com/subject/1787291/
https://movie.douban.com/subject/10577869/
https://movie.douban.com/subject/1395091/
https://movie.douban.com/subject/11026735/
https://movie.douban.com/subject/4202302/
https://movie.douban.com/subject/1291990/
https://movie.douban.com/subject/27010768/
https://movie.douban.com/subject/1297447/
https://movie.douban.com/subject/1292337/
https://movie.douban.com/subject/1291999/
https://movie.douban.com/subject/1297359/
https://movie.douban.com/subject/1292370/
https://movie.douban.com/subject/1293359/
https://movie.douban.com/subject/26799731/
https://movie.douban.com/subject/1296339/
https://movie.douban.com/subject/1300992/
https://movie.douban.com/subject/1297630/
https://movie.douban.com/subject/1291875/
https://movie.douban.com/subject/4268598/
https://movie.douban.com/subject/1309055/
https://movie.douban.com/subject/1292215/
https://movie.douban.com/subject/6307447/
https://movie.douban.com/subject/1303037/
https://movie.douban.com/subject/10533913/
https://movie.douban.com/subject/10437779/
https://movie.douban.com/subject/3287562/
https://movie.douban.com/subject/25814707/
https://movie.douban.com/subject/26628357/
https://movie.douban.com/subject/1828115/
https://movie.douban.com/subject/25986180/
https://movie.douban.com/subject/25814705/
https://movie.douban.com/subject/2353023/
https://movie.douban.com/subject/1294240/
https://movie.douban.com/subject/1297518/
https://movie.douban.com/subject/1297052/
https://movie.douban.com/subject/1291579/
https://movie.douban.com/subject/5989818/
https://movie.douban.com/subject/1307315/
https://movie.douban.com/subject/1418200/
https://movie.douban.com/subject/1858711/
https://movie.douban.com/subject/3072124/
https://movie.douban.com/subject/1461403/
https://movie.douban.com/subject/1292274/
https://movie.douban.com/subject/21360417/
https://movie.douban.com/subject/1793929/
https://movie.douban.com/subject/1578507/
https://movie.douban.com/subject/24733428/
https://movie.douban.com/subject/10463953/
https://movie.douban.com/subject/1417598/
https://movie.douban.com/subject/1292208/
https://movie.douban.com/subject/1293181/
https://movie.douban.com/subject/1302467/
https://movie.douban.com/subject/1316510/
https://movie.douban.com/subject/27119724/
https://movie.douban.com/subject/26325320/
https://movie.douban.com/subject/4920389/
https://movie.douban.com/subject/4917726/
https://movie.douban.com/subject/1307856/
https://movie.douban.com/subject/25917973/
https://movie.douban.com/subject/1306861/
https://movie.douban.com/subject/1300374/
https://movie.douban.com/subject/26683290/
https://movie.douban.com/subject/26611804/
https://movie.douban.com/subject/1291557/
https://movie.douban.com/subject/1907966/
https://movie.douban.com/subject/1297574/
https://movie.douban.com/subject/1295399/
https://movie.douban.com/subject/1291545/
https://movie.douban.com/subject/1291822/
https://movie.douban.com/subject/1293460/
https://movie.douban.com/subject/1292329/
https://movie.douban.com/subject/1292226/
https://movie.douban.com/subject/1307106/
https://movie.douban.com/subject/1304447/
https://movie.douban.com/subject/1291585/
https://movie.douban.com/subject/1295409/
https://movie.douban.com/subject/2209573/
https://movie.douban.com/subject/1959195/
https://movie.douban.com/subject/1867345/
https://movie.douban.com/subject/10808442/
https://movie.douban.com/subject/25773932/
https://movie.douban.com/subject/24750126/
https://movie.douban.com/subject/3792799/
https://movie.douban.com/subject/1291578/
https://movie.douban.com/subject/27622447/
https://movie.douban.com/subject/1291992/
https://movie.douban.com/subject/1307811/
https://movie.douban.com/subject/27059130/
https://movie.douban.com/subject/25895901/
https://movie.douban.com/subject/6534248/
https://movie.douban.com/subject/1862151/
https://movie.douban.com/subject/1310177/
https://movie.douban.com/subject/3011051/
https://movie.douban.com/subject/1291879/
https://movie.douban.com/subject/1292401/
https://movie.douban.com/subject/3592854/
https://movie.douban.com/subject/4739952/
https://movie.douban.com/subject/1438652/
https://movie.douban.com/subject/1308857/
https://movie.douban.com/subject/1303394/
https://movie.douban.com/subject/1297880/
https://movie.douban.com/subject/25934014/
https://movie.douban.com/subject/1293908/
https://movie.douban.com/subject/5300054/
https://movie.douban.com/subject/25980443/
https://movie.douban.com/subject/1959877/
https://movie.douban.com/subject/1422283/
https://movie.douban.com/subject/3075287/
https://movie.douban.com/subject/1309163/
https://movie.douban.com/subject/4848115/
https://movie.douban.com/subject/1291844/
https://movie.douban.com/subject/1978709/
https://movie.douban.com/subject/34874432/
https://movie.douban.com/subject/1292287/
https://movie.douban.com/subject/1296909/
https://movie.douban.com/subject/1291870/
https://movie.douban.com/subject/1302827/
https://movie.douban.com/subject/1292328/
https://movie.douban.com/subject/1428175/
https://movie.douban.com/subject/2213597/
https://movie.douban.com/subject/1305690/
https://movie.douban.com/subject/1419936/
https://movie.douban.com/subject/1308767/
https://movie.douban.com/subject/26430107/
https://movie.douban.com/subject/1304141/
https://movie.douban.com/subject/2051007/
https://movie.douban.com/subject/25724855/
https://movie.douban.com/subject/3552028/
https://movie.douban.com/subject/1300117/
https://movie.douban.com/subject/1293323/
https://movie.douban.com/subject/25807345/
https://movie.douban.com/subject/2222996/
https://movie.douban.com/subject/25864085/
https://movie.douban.com/subject/1307394/
https://movie.douban.com/subject/26374197/
https://movie.douban.com/subject/1293318/
https://movie.douban.com/subject/1304102/
https://movie.douban.com/subject/1293964/
https://movie.douban.com/subject/26615208/
https://movie.douban.com/subject/26787574/
https://movie.douban.com/subject/4202982/
https://movie.douban.com/subject/1457217/
https://movie.douban.com/subject/2133323/
相关推荐
binishuaio几秒前
Java 第11天 (git版本控制器基础用法)
java·开发语言·git
zz.YE2 分钟前
【Java SE】StringBuffer
java·开发语言
就是有点傻6 分钟前
WPF中的依赖属性
开发语言·wpf
洋24015 分钟前
C语言常用标准库函数
c语言·开发语言
进击的六角龙16 分钟前
Python中处理Excel的基本概念（如工作簿、工作表等）
开发语言·python·excel
wrx繁星点点18 分钟前
状态模式（State Pattern）详解
java·开发语言·ui·设计模式·状态模式
NoneCoder35 分钟前
Java企业级开发系列（1）
java·开发语言·spring·团队开发·开发
苏三有春35 分钟前
PyQt5实战——UTF-8编码器功能的实现（六）
开发语言·qt
一只爱好编程的程序猿37 分钟前
Java后台生成指定路径下创建指定名称的文件
java·python·数据下载
Aniay_ivy42 分钟前
深入探索 Java 8 Stream 流：高效操作与应用场景
java·开发语言·python
热门推荐
01【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总02(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明03组基轨迹建模 GBTM的介绍与实现（Stata 或 R）04【AIGC】重塑未来的科技巨轮05全面解析：构建基于深度学习的安全帽检测系统（UI界面+YOLO代码+数据集）06【经验分享】Ubuntu22.04安装微信(linux官方版)07基于YOLOv10深度学习的CT扫描图像肾结石智能检测系统【python源码+Pyqt5界面+数据集+训练代码】深度学习实战、目标检测08Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO09RAG 实践- Ollama+RagFlow 部署本地知识库10红米手机使用google play