python连接mysql,并整理(去哪儿网)页面数据到表

##引入requests/pymysql模块

本地安装mysql数据库,安装图形化工具navicat

python 复制代码
import requests
from pymysql import Connect

#创建客户端连接信息
client = Connect(
    host='127.0.0.1',
    port=3306,
    user='root',
    password='',
)
#创建游标
cursor = client.cursor()
cursor.execute('create database if not exists qunar')
cursor.execute('use qunar')
cursor.execute('create table if not exists qunar_hotel('
               'id int primary key auto_increment,'
               'name char(100),'
               'city_name char(30),'
               'price int,'
               'score int,'
               'type char(60),'
               'address char(255),'
               'comment char(60))')

#引入城市列表
city_url_list = [{
    "name": "北京",
    "url": "beijing_city"
}, {
    "name": "成都",
    "url": "chengdu"
}, {
    "name": "广州",
    "url": "guangzhou"
}, {
    "name": "杭州",
    "url": "hangzhou"
}, {
    "name": "昆明",
    "url": "kunming"
}, {
    "name": "青岛",
    "url": "qingdao"
}, {
    "name": "上海",
    "url": "shanghai_city"
}, {
    "name": "深圳",
    "url": "shenzhen"
}, {
    "name": "西安",
    "url": "xian"
}, {
    "name": "重庆",
    "url": "chongqing_city"
}, {
    "name": "北海",
    "url": "beihai"
}, {
    "name": "大理",
    "url": "dali"
}, {
    "name": "大连",
    "url": "dalian"
}, {
    "name": "东莞",
    "url": "dongguan"
}, {
    "name": "佛山",
    "url": "foshan"
}, {
    "name": "福州",
    "url": "fuzhou_fujian"
}, {
    "name": "贵阳",
    "url": "guiyang"
}, {
    "name": "桂林",
    "url": "guilin"
}, {
    "name": "哈尔滨",
    "url": "haerbin"
}, {
    "name": "海口",
    "url": "haikou"
}, {
    "name": "合肥",
    "url": "hefei"
}, {
    "name": "惠州",
    "url": "huizhou_guangdong"
}, {
    "name": "济南",
    "url": "jinan"
}, {
    "name": "嘉兴",
    "url": "jiaxing"
}, {
    "name": "兰州",
    "url": "lanzhou"
}, {
    "name": "丽江",
    "url": "lijiang"
}, {
    "name": "南昌",
    "url": "nanchang"
}, {
    "name": "南京",
    "url": "nanjing"
}, {
    "name": "南宁",
    "url": "nanning"
}, {
    "name": "宁波",
    "url": "ningbo"
}, {
    "name": "黔东南",
    "url": "qiandongnan"
}, {
    "name": "秦皇岛",
    "url": "qinhuangdao"
}, {
    "name": "泉州",
    "url": "quanzhou"
}, {
    "name": "三亚",
    "url": "sanya"
}, {
    "name": "厦门",
    "url": "xiamen"
}, {
    "name": "沈阳",
    "url": "shenyang"
}, {
    "name": "石家庄",
    "url": "shijiazhuang"
}, {
    "name": "苏州",
    "url": "suzhou_jiangsu"
}, {
    "name": "太原",
    "url": "taiyuan"
}, {
    "name": "天津",
    "url": "tianjin_city"
}, {
    "name": "威海",
    "url": "weihai"
}, {
    "name": "温州",
    "url": "wenzhou"
}, {
    "name": "乌鲁木齐",
    "url": "wulumuqi"
}, {
    "name": "无锡",
    "url": "wuxi"
}, {
    "name": "武汉",
    "url": "wuhan"
}, {
    "name": "西宁",
    "url": "xining"
}, {
    "name": "烟台",
    "url": "yantai"
}, {
    "name": "长沙",
    "url": "changsha"
}, {
    "name": "郑州",
    "url": "zhengzhou"
}, {
    "name": "珠海",
    "url": "zhuhai"
}]

for city_dict in city_url_list[:10]:
    url = f'https://hotel.qunar.com/napi/hotHotelList?size=150&city={city_dict["url"]}'
    headers = {
       'user-agent':'https://hotel.qunar.com/napi/hotHotelList?size=150&city={city_dict["url"]}'
    }
    res = requests.get(url,headers=headers)
    res
    Data = tuple(map(lambda x:(x.get('name'),
                               x.get('cityName'),
                               eval(x.get('price')),
                               eval(x.get('score')),
                               x.get('hotelCategory') if x.get('hotelCategory') else 'null',
                               x.get('locationInfo'),
                               x.get('commentDesc') if x.get('commentDesc') else 'null',
                               ),res.json()))

    insert_sql = 'insert into qunar_hotel(name,city_name,price,score,type,address,comment) values(%s,%s,%s,%s,%s,%s,%s)'
    cursor.executemany(insert_sql,Data)

    # 提交事务
    client.commit()
else:
    cursor.close()
    client.close()

运行结果:

相关推荐
倔强的石头_15 分钟前
你的 Python 为什么“优雅地慢”?——读《极速Python:高性能编码、计算与数据分析》
python
程序猿 小项目大搞头22 分钟前
视频加水印,推荐使用运营大管家-视频批量加水印软件
python
安然~~~39 分钟前
mysql的索引
数据库·mysql
Adorable老犀牛44 分钟前
可遇不可求的自动化运维工具 | 2 | 实施阶段一:基础准备
运维·git·vscode·python·node.js·自动化
SeaTunnel1 小时前
实测有效|用 SeaTunnel 免费实现 MySQL→Oracle 实时同步,步骤超细
数据库·mysql·oracle·开源·seatunnel·数据同步·连接器
咔咔一顿操作1 小时前
MySQL 事务管理与锁机制:解决并发场景下的数据一致性问题
java·数据库·mysql
xchenhao1 小时前
SciKit-Learn 全面分析 digits 手写数据集
python·机器学习·分类·数据集·scikit-learn·svm·手写
胡耀超2 小时前
7、Matplotlib、Seaborn、Plotly数据可视化与探索性分析(探索性数据分析(EDA)方法论)
python·信息可视化·plotly·数据挖掘·数据分析·matplotlib·seaborn
tangweiguo030519872 小时前
Django REST Framework 构建安卓应用后端API:从开发到部署的完整实战指南
服务器·后端·python·django
Dfreedom.2 小时前
在Windows上搭建GPU版本PyTorch运行环境的详细步骤
c++·人工智能·pytorch·python·深度学习