python读取excel数据写入mysql

概述

业务中有时会需要解析excel中的数据,按照要求处理后,写入到db中;

python处理这个正好简便快捷

demo

没有依赖就 pip install pymysql一下

python 复制代码
import pymysql
from pymysql.converters import escape_string
from openpyxl import load_workbook
from Snowflake import Snowflake


def load_excel_data(snowflake):
    # 连接到MySQL数据库
    mydb = pymysql.connect(
        host="xxx.xxx.xxx.xxx",
        port=3306,
        user="xxx",
        passwd="xxx",
        db="xxxx"
    )

    # 打开Excel文件
    wb = load_workbook(filename=r'D:\xx\test.xlsx')
    sheet = wb.active

    # 获取表头
    header = [cell.value for cell in sheet[1]]

    column_header = []
	# 表头转换列名
    for excel_head_name in header:
        if '11' == excel_head_name:
            column_header.append("xx")
        elif '22' == excel_head_name:
            column_header.append("xx")
        elif '33' == excel_head_name:
            column_header.append("xx")
        elif '1122' == excel_head_name:
            column_header.append("xx")


    # 遍历每一行数据,并将其插入到数据库中
    cursor = mydb.cursor()
    count = 0

    defaultUser = "'xxx'"

    for row in sheet.iter_rows(min_row=2, values_only=True):
        cId = snowflake.next_id()

        date = row[0]
        # datetime 转 date
        date = date.date()

        a2 = row[1]
        reason = row[2]
        detail = row[3]
		
		# \'%s\' 将含有特殊内容的字符串整个塞进去
        sql = f"INSERT INTO test_table (id, store_id, num, handler, create_by, update_by, date, a2, reason, detail) VALUES ({cId}, 3, 0, 43, {defaultUser}, {defaultUser}, \'%s\', \'%s\', \'%s\', \'%s\')" % (date, self_escape_string(a2), self_escape_string(reason), self_escape_string(detail))

        print(sql)

        # cursor.execute(sql, row)
        cursor.execute(sql)
        count += 1
        print(f"正在插入{count}条数据")

    # 提交更改并关闭数据库连接
    mydb.commit()
    cursor.close()
    mydb.close()

# 将字符串中的特殊字符转义
# python中没有null只有None
def self_escape_string(data):
    if data is None:
        return ""
    return escape_string(data)



if __name__ == '__main__':
    worker_id = 1
    data_center_id = 1
    snowflake = Snowflake(worker_id, data_center_id)

    load_excel_data(snowflake)

雪花id生成主键

python 复制代码
import time
import random


class Snowflake:
    def __init__(self, worker_id, data_center_id):
        ### 机器标识ID
        self.worker_id = worker_id
        ### 数据中心ID
        self.data_center_id = data_center_id
        ### 计数序列号
        self.sequence = 0
        ### 时间戳
        self.last_timestamp = -1

    def next_id(self):
        timestamp = int(time.time() * 1000)
        if timestamp < self.last_timestamp:
            raise Exception(
                "Clock moved backwards. Refusing to generate id for %d milliseconds" % abs(timestamp - self.last_timestamp))
        if timestamp == self.last_timestamp:
            self.sequence = (self.sequence + 1) & 4095
            if self.sequence == 0:
                timestamp = self.wait_for_next_millis(self.last_timestamp)
        else:
            self.sequence = 0
        self.last_timestamp = timestamp
        return ((timestamp - 1288834974657) << 22) | (self.data_center_id << 17) | (self.worker_id << 12) | self.sequence



    def next_id(self):
        timestamp = int(time.time() * 1000)
        if timestamp < self.last_timestamp:
            raise Exception("Clock moved backwards. Refusing to generate id for %d milliseconds" % abs(timestamp - self.last_timestamp))
        if timestamp == self.last_timestamp:
            self.sequence = (self.sequence + 1) & 4095
            if self.sequence == 0:
                timestamp = self.wait_for_next_millis(self.last_timestamp)
        else:
            self.sequence = 0
        self.last_timestamp = timestamp
        return ((timestamp - 1288834974657) << 22) | (self.data_center_id << 17) | (self.worker_id << 12) | self.sequence

    def wait_for_next_millis(self, last_timestamp):
        timestamp = int(time.time() * 1000)
        while timestamp <= last_timestamp:
            timestamp = int(time.time() * 1000)
        return timestamp
相关推荐
斑点鱼 SpotFish1 分钟前
用Python可视化国庆期间旅游概况与消费趋势
开发语言·python·旅游
小兔崽子去哪了4 分钟前
Python 学习记录
python
埃泽漫笔24 分钟前
RabbitMQ四种交换机详解
python·mq
小猪快跑爱摄影1 小时前
【附代码】Jupyter 多进程调用 seaborn 并保留格式
python·jupyter
技术猴小猴1 小时前
如何使用Python实现LRU缓存
python·spring·缓存
Java陈序员1 小时前
简单好用!一款针对 IT 团队开发的文档管理系统!
mysql·docker·go
2401_841495641 小时前
【自然语言处理】“bert-base-chinese”的基本用法及实战案例
人工智能·python·自然语言处理·bert·文本预处理·特征提取·训练验证
缺点内向1 小时前
C# 中 Excel 工作表打印前页面边距的设置方法
c#·.net·excel
猫头虎1 小时前
AI_NovelGenerator:自动化长篇小说AI生成工具
运维·人工智能·python·自动化·aigc·gpu算力·ai-native
l1t2 小时前
DeepSeek辅助测试三种ODS电子表格写入程序
python·xlsx·ods·deepseek·xlsb