Python爬虫使用示例-古诗词摘录

镜花照无眠2024-10-19 20:10

一、分析需求

目标地址：

复制代码 
https://www.sou-yun.cn/Query.aspx?type=poem&id=×××××

二、提取诗句

python 复制代码 
import os
import re
import requests
import parsel

#url ='https://www.sou-yun.cn/PoemIndex.aspx?dynasty=Tang&author=14976&type=Jie'
url='https://www.sou-yun.cn/Query.aspx?type=poem1&id=36647'
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url,headers=headers)
html_content= response.text
#print(response.text)
# 正则表达式匹配
poem_sentences = re.findall(r"<div class='poemSentence'[^>]*>(.*?)<\/div>", html_content, re.DOTALL)

# 清理并输出提取的诗句
for sentence in poem_sentences:
    # 移除HTML标签
    clean_sentence = re.sub(r"<.*?>", "", sentence).strip()
    if clean_sentence:  # 过滤掉空句
        print(clean_sentence)

三、其他信息

提取all需要信息，title+author+sentences

python 复制代码 
import os
import re
import requests
import parsel

#url ='https://www.sou-yun.cn/PoemIndex.aspx?dynasty=Tang&author=14976&type=Jie'
url='https://www.sou-yun.cn/Query.aspx?type=poem1&id=36647'
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url,headers=headers)
html_content= response.text
#print(response.text)
# 提取标题
title_match = re.search(r"<span class='bold'><span class='wordLink'[^>]*>(.*?)<\/span><\/span>\s*<span[^>]*>(.*?)<\/span>\s*<span class='poemAuthor'[^>]*>(.*?)<\/span>", html_content)
if title_match:
    title = title_match.group(1) + title_match.group(2)  # 合并标题部分
    author = re.sub(r"<.*?>", "", title_match.group(3)).strip()  # 处理作者

# 正则表达式匹配诗句
poem_sentences = re.findall(r"<div class='poemSentence'[^>]*>(.*?)<\/div>", html_content, re.DOTALL)

# 清理并输出提取的信息
print("标题:", title)
print("作者:", author)
print("诗句:")

for sentence in poem_sentences:
    # 移除HTML标签
    clean_sentence = re.sub(r"<.*?>", "", sentence).strip()
    if clean_sentence:  # 过滤掉空句
        print(clean_sentence)

微调格式

python 复制代码 
import os
import re
import requests
import parsel

#url ='https://www.sou-yun.cn/PoemIndex.aspx?dynasty=Tang&author=14976&type=Jie'
url='https://www.sou-yun.cn/Query.aspx?type=poem1&id=36647'
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url,headers=headers)
html_content= response.text
#print(response.text)
# 提取标题
title_match = re.search(r"<span class='bold'><span class='wordLink'[^>]*>(.*?)<\/span><\/span>\s*<span[^>]*>(.*?)<\/span>\s*<span class='poemAuthor'[^>]*>(.*?)<\/span>", html_content)
if title_match:
    title = title_match.group(1) + title_match.group(2)  # 合并标题部分
    author = re.sub(r"<.*?>", "", title_match.group(3)).strip()  # 处理作者

# 正则表达式匹配诗句
poem_sentences = re.findall(r"<div class='poemSentence'[^>]*>(.*?)<\/div>", html_content, re.DOTALL)

# 清理并输出提取的信息
print("《 " + title + "》 （"+ author + "）")
#print("作者:", author)
#print("诗句:")

for sentence in poem_sentences:
    # 移除HTML标签
    clean_sentence = re.sub(r"<.*?>", "", sentence).strip()
    if clean_sentence:  # 过滤掉空句
        print(clean_sentence)

四、保存文档

保存到txt里面,单首诗歌

python 复制代码 
import os
import re
import requests
import parsel


#url ='https://www.sou-yun.cn/PoemIndex.aspx?dynasty=Tang&author=14976&type=Jie'
url='https://www.sou-yun.cn/Query.aspx?type=poem1&id=36647'
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
response = requests.get(url=url,headers=headers)
html_content= response.text
#print(response.text)
# 提取标题
title_match = re.search(r"<span class='bold'><span class='wordLink'[^>]*>(.*?)<\/span><\/span>\s*<span[^>]*>(.*?)<\/span>\s*<span class='poemAuthor'[^>]*>(.*?)<\/span>", html_content)
if title_match:
    title = title_match.group(1) + title_match.group(2)  # 合并标题部分
    author = re.sub(r"<.*?>", "", title_match.group(3)).strip()  # 处理作者

# 正则表达式匹配诗句
poem_sentences = re.findall(r"<div class='poemSentence'[^>]*>(.*?)<\/div>", html_content, re.DOTALL)

# 清理并准备写入文件的内容
output = f"《 " + title + "》 （"+ author + "）\n"
print("《 " + title + "》 （"+ author + "）")

for sentence in poem_sentences:
    # 移除HTML标签
    clean_sentence = re.sub(r"<.*?>", "", sentence).strip()
    if clean_sentence:  # 过滤掉空句
        output += clean_sentence + "\n"
        print(clean_sentence)

# 将结果写入文本文件

with open('poem.txt', 'w', encoding='utf-8') as file:
    file.write(output)

print("信息已保存到 poem.txt")

五、多首继续

不一定是符合要求的，因为这个id暂时得不到(内容结构问题)

python 复制代码 
import os
import re
import requests
import parsel


#url ='https://www.sou-yun.cn/PoemIndex.aspx?dynasty=Tang&author=14976&type=Jie'
#url='https://www.sou-yun.cn/Query.aspx?type=poem1&id=36647'
#headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
#response = requests.get(url=url,headers=headers)
#html_content= response.text
#print(response.text)

# 指定保存文件的路径
output_file_path = 'all_poems.txt'

# 先清空（如果存在）或创建目标文件
with open(output_file_path, 'w', encoding='utf-8') as file:
    file.write("")  # 清空文件内容

# 循环下载每首诗
for poem_id in range(36647, 36848):
    url = f'https://www.sou-yun.cn/Query.aspx?type=poem1&id={poem_id}'
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.139 Safari/537.36'}
    response = requests.get(url=url, headers=headers)
    #html_content = response.text
    # 获取网页内容
    #response = requests.get(url)

    if response.status_code == 200:
        html_content = response.text

        # 提取标题
        title_match = re.search(
            r"<span class='bold'><span class='wordLink'[^>]*>(.*?)<\/span><\/span>\s*<span[^>]*>(.*?)<\/span>\s*<span class='poemAuthor'[^>]*>(.*?)<\/span>",
            html_content)
        if title_match:
            title = title_match.group(1) + title_match.group(2)  # 合并标题部分
            author = re.sub(r"<.*?>", "", title_match.group(3)).strip()  # 处理作者

            # 正则表达式匹配诗句
            poem_sentences = re.findall(r"<div class='poemSentence'[^>]*>(.*?)<\/div>", html_content, re.DOTALL)

            # 清理并准备写入文件的内容
            output = f"《 " + title + "》 （"+ author + "）\n"

            for sentence in poem_sentences:
                # 移除HTML标签
                clean_sentence = re.sub(r"<.*?>", "", sentence).strip()
                if clean_sentence:  # 过滤掉空句
                    output += clean_sentence + "\n"

            # 为每首诗添加分隔线
            output += "\n" + "=" * 50 + "\n\n"  # 分隔线，用于区分不同的诗

            # 将结果追加到文本文件
            with open(output_file_path, 'a', encoding='utf-8') as file:  # 以追加模式打开文件
                file.write(output)

            print(f"信息已保存到 {output_file_path}")
        else:
            print(f"在ID {poem_id} 的页面中找不到诗的标题或作者。")
    else:
        print(f"无法获取ID {poem_id} 的页面，状态码: {response.status_code}")

运行结果：

相关推荐
He1955015 分钟前
Go初级之十：错误处理与程序健壮性
开发语言·python·golang
m0_7381207230 分钟前
CTFshow系列——PHP特性Web93-96
开发语言·安全·web安全·php·ctfshow
m0_5704664141 分钟前
代码随想录算法训练营第二十八天 | 买卖股票的最佳实际、跳跃游戏、K次取反后最大化的数组和
java·开发语言·算法
程序喵大人41 分钟前
分享个C++线程池的实现源码
开发语言·c++·线程池
和鲸社区1 小时前
《斯坦福CS336》作业1开源，从0手搓大模型｜代码复现+免环境配置
人工智能·python·深度学习·计算机视觉·语言模型·自然语言处理·nlp
不会吃萝卜的兔子1 小时前
go webrtc - 1 go基本概念
开发语言·golang·webrtc
豌豆花下猫1 小时前
Python 潮流周刊#118：Python 异步为何不够流行？（摘要）
后端·python·ai
THMAIL1 小时前
深度学习从入门到精通 - LSTM与GRU深度剖析：破解长序列记忆遗忘困境
人工智能·python·深度学习·算法·机器学习·逻辑回归·lstm
要做朋鱼燕1 小时前
【C++】 priority_queue 容器模拟实现解析
开发语言·c++·笔记·职场和发展
jiaway2 小时前
【C语言】第四课 指针与内存管理
c语言·开发语言·算法
热门推荐
012025 年高教社杯全国大学生数学建模竞赛C 题 NIPT 的时点选择与胎儿的异常判定 完整成品思路模型代码分享，全网首发高质量！！！022025年数学建模国赛C题超详细解题思路032025全国大学生数学建模C题保姆级思路模型（持续更新）：NIPT 的时点选择与胎儿的异常判定042025高教社杯国赛数学建模选题建议+初步分析052025国赛B题保姆级教程思路分析 碳化硅外延层厚度的确定06UV安装并设置国内源07奈飞工厂官网，国内Netflix影视在线看|中文网页电脑版入口08KGG转MP3工具|非KGM文件|解密音频09（E题｜AI 辅助智能体测）2025年高教杯全国大学生数学建模国赛解题思路|完整代码论文集合10不再让Windows更新！&Edge游戏助手卸载及关闭自动更新