通过GitHub探索Python爬虫技术

fyhs2024-03-05 16:48

1.检索爬取内容案例。

2.找到最近更新的。(最新一般都可以直接运行)

3.选择适合自己的项目，目前测试下面画红圈的是可行的。

4.方便大家查看就把代码粘贴出来了。

python 复制代码 
#图中画圈一代码
import requests
import os
import re


while True:
    music_id = input("请输入歌曲id或歌曲链接: ")
    if music_id.startswith("http"):
        music_id = re.search(r"id=(\d+)", music_id).group(1)
    get_lyric = requests.get(url="https://music.163.com/api/song/lyric", params={"id": music_id, "lv": 1, "kv": 1, "tv": -1}).json()
    print(get_lyric)
    if get_lyric.get("lrc").get("lyric") == "":
        print("该歌曲没有歌词")
    else:
        if not os.path.exists("./OutLyric"):
            os.makedirs("./OutLyric")
        with open(f"./OutLyric/{music_id}.lrc", "w", encoding="utf-8") as save_lyric:
            if get_lyric.get("tlyric").get("lyric") == "":
                save_lyric.write(get_lyric.get("lrc").get("lyric"))
            else:
                zh_cn_lyric = re.sub(r'\[[^0-9]*:[^0-9.]*]\n', '', get_lyric.get("tlyric").get("lyric"))
                save_lyric.write(f'{get_lyric.get("lrc").get("lyric")}\n{zh_cn_lyric}')
        print(f"下载成功,可将该文件重命名至与歌曲相同的名字使用,lrc文件保存至./OutLyric/{music_id}.lrc")
python 复制代码 
#图中画圈2代码
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import requests as rq
from requests import exceptions
from bs4 import BeautifulSoup as BS
import os
import re
import csv


SONG_NUM = 0


def getMusic(ID, path, num):
    cloud = 'http://music.163.com/song/media/outer/url?id='
    kv = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36'}
    try:
        url = cloud+ID+'.mp3'
        tmp = rq.get(url, headers=kv)
        tmp.raise_for_status()
        print(num+"、歌曲正在下载...")
        with open(path, 'wb') as f:
            f.write(tmp.content)
        f.close()
        print(num+"、歌曲下载成功！")
    except exceptions.HTTPError as e:
        print(e)
    except Exception as e:
        print(e)


def getMusicText(ID, path, num):

    muTextUrl = 'http://music.163.com/api/song/lyric?id=' + ID + '&lv=1&kv=1&tv=-1'
    headers = {
        'Referer': 'https://music.163.com',
        'Host': 'music.163.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
    }
    try:
        res = rq.get(muTextUrl, headers=headers)
        res.raise_for_status()
        false = False  # 解决eval报错 name 'false' is not defined
        true = True
        null = None
        lrc_dict = eval(res.text)  # 转换为dict字典
        lrc_dict = lrc_dict['lrc']
        music_lyric = lrc_dict['lyric']
        print(num+"、歌词正在下载...")
        with open(path, 'w', encoding="utf-8") as f:
            f.write(music_lyric)
        f.close()
        print(num+"、歌词下载成功！")
    except exceptions.HTTPError as e:
        print(e)
    except Exception as e:
        print(e)


def create_csv_head():
    headers = ['song_num', 'song_name', 'singer', 'song_duration']
    with open("./music/musicMsg.csv", "a", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=headers)
        head = {'song_num': '榜单序号', 'song_name': '歌曲名称',
                'singer': '歌手', 'song_duration': '歌曲时长'}
        writer.writerow(head)


def save_musicMsg(music_dict):
    headers = ['song_num', 'song_name', 'singer', 'song_duration']
    with open("./music/musicMsg.csv", "a", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=headers)
        writer.writerow(music_dict)


def split_Msg(msg):
    msg = msg.split('"')
    item = msg[1]
    return item


def getMusicMsg(ID):
    global SONG_NUM
    song_url = 'https://music.163.com/song?id=' + ID
    headers = {
        'Referer': 'https://music.163.com',
        'Host': 'music.163.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
    }
    try:
        s = rq.session()
        res = s.get(song_url, headers=headers)
        soup = BS(res.content, 'lxml')
        # 获取歌手
        singer = str(soup.find('meta', {'property': 'og:music:artist'}))
        singer = split_Msg(singer)
        # 获取歌曲名
        song_name = str(soup.find('meta', {'property': 'og:title'}))
        song_name = split_Msg(song_name)
        # 获取歌曲时长
        song_duration = str(soup.find('meta', {'property': 'music:duration'}))
        song_duration = split_Msg(song_duration)
        m, s = divmod(int(song_duration), 60)
        song_duration = ("%02d:%02d" % (m, s))
        music_dict = {
            'song_num': SONG_NUM,
            'song_name': song_name,
            'singer': singer,
            'song_duration': song_duration
        }
        save_musicMsg(music_dict)
        # 歌曲名中/\\替换为空
        if '/' in song_name or '\\' in song_name or ':' in song_name:
            song_name = song_name.replace('/', '')
            song_name = song_name.replace('\\', '')
            song_name = song_name.replace(':', '')
        # 歌手名中/\\替换为&
        if '/' in singer or '\\' in singer or ':' in singer:
            singer = singer.replace('/', '&')
            singer = singer.replace('\\', '&')
            singer = singer.replace(':', '')
        dirName = singer+'-'+song_name
        print(dirName)

        return dirName
    except exceptions.HTTPError as e:
        print(e)
    except Exception as e:
        print(e)


def getMusicList():
    headers = {
        'Referer': 'https://music.163.com',
        'Host': 'music.163.com',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'
    }
    base_url = 'https://music.163.com/discover/toplist'
    s = rq.session()
    url = base_url
    response = s.get(url, headers=headers)
    soup = BS(response.content, "lxml")
    main = soup.find('ul', {'class': 'f-hide'})
    ls = main.find_all('a')
    songID_dic = {}  # key song_name ,value songID
    print('一共有'+str(len(ls))+'首歌')
    a = 1
    for music in ls:
        name = music.text
        ID = str(music['href'].replace('/song?id=', ''))
        name = name+'_'+str(a)
        a += 1
        songID_dic[name] = ID
        print("Name:{:30}\tID{:^10}".format(name, ID))
    print('一共有'+str(len(songID_dic))+'')
    return songID_dic


def main():
    global SONG_NUM
    songID_dic = getMusicList()
    rootDir = 'music'
    if os.path.exists(rootDir):
        print(rootDir+"文件夹已存在")
    else:
        os.mkdir(rootDir)
        print("创建文件夹"+rootDir)
    create_csv_head()
    for item in songID_dic:
        item_clear = item.split('_')[0]
        SONG_NUM += 1
        dirName = getMusicMsg(songID_dic[item])
        if dirName[-2:-1] == '.':
            dirName = dirName.replace('.', '·')
        musicDir = './'+rootDir+'/' + dirName
        if os.path.exists(musicDir):
            print(musicDir+"文件夹已存在")
        else:
            os.mkdir(musicDir)
        print("创建文件夹"+musicDir)
        if len(item_clear) > 75:
            item_clear = item_clear[:70]+'···'
        elif '.' in item_clear:
            item_clear = item_clear.replace('.', '·')
        print(item_clear, end="    \n")
        mp3_path = musicDir+'/'+item_clear+'.mp3'
        m4a_path = musicDir+'/'+item_clear+'.m4a'
        lyric_path = musicDir+'/'+item_clear+'.txt'
        num = str(SONG_NUM)
        print('='*50)
        getMusic(songID_dic[item], mp3_path, num)
        getMusic(songID_dic[item], m4a_path, num)
        print('*'*50)
        getMusicText(songID_dic[item], lyric_path, num)
        print('='*50)


if __name__ == '__main__':
    main()
    # getMusicList()
    # getMusicText("1994955842", "path")
    # getMusicMsg("1998931166")
相关推荐
学习路上_write11 分钟前
FPGA/Verilog,Quartus环境下if-else语句和case语句RT视图对比/学习记录
单片机·嵌入式硬件·qt·学习·fpga开发·github·硬件工程
油泼辣子多加13 小时前
2024年11月12日Github流行趋势
github
wayuncn13 小时前
IDC机房服务器托管的费用组成
运维·服务器·github
惜.己16 小时前
Jmeter中的配置原件（五）
功能测试·测试工具·jmeter·github·1024程序员节
Vicky&James17 小时前
英雄联盟客户端项目：从跨平台Uno Platform到Win UI3的转换只需要30分钟
github·wpf·跨平台·英雄联盟·winui·unoplatform
一个处女座的程序猿21 小时前
LLMs之Code：Github Spark的简介、安装和使用方法、案例应用之详细攻略
大数据·spark·github
梓羽玩Python1 天前
5.3K Star 烦恼全丢掉！GlazeWM：让Windows下的窗口管理更高效的秘密武器！
开源·github
shiming88791 天前
GitHub Org如何运营
github
雪峰1 天前
使用 Web Search 插件扩展 GitHub Copilot 问答
vscode·github·copilot
油泼辣子多加2 天前
2024年11月11日Github流行趋势
github
热门推荐
01软件工程从理论到实践客观题汇总(头歌第一章至第八章)02【HarmonyOS】HUAWEI DevEco Studio 下载地址汇总03组基轨迹建模 GBTM的介绍与实现（Stata 或 R）04VSCode插件 —— Cody AI （免费AI助手！）05全面解析：构建基于深度学习的安全帽检测系统（UI界面+YOLO代码+数据集）06Windows10安装PCL1.14.0及点云配准07Ubuntu 20.04使用Livox mid 360 测试 FAST_LIO08文件或文件夹名称中有空格如何批量去除09基于YOLOv10深度学习的CT扫描图像肾结石智能检测系统【python源码+Pyqt5界面+数据集+训练代码】深度学习实战、目标检测10【QuikGraph】C#调用第三方库实现迪杰斯特拉(Dijkstra)算法功能