python 常用功能积累

sql查询

python 复制代码
import pymysql
import pandas as pd

user = '' #用户名
password = '' #密码
dbName = ''  #库名
dbHost = '' #ip
dbPort = 8888
con = pymysql.connect(
    host=dbHost,
    port=dbPort,
    user=user,
    password=password,
    database=dbName,
    charset='utf8')
cursor = con.cursor()
head = ["Id", "Url"]
t0, t1, name = '', '', ''
sql_select = "SELECT id, Url " \
             "FROM xxx " \
             "WHERE createTime >= ('{}') and createTime <= ('{}') and name = ('{}')".format(t0, t1, name)
cursor.execute(sql_select)
cds = cursor.fetchall()
df = pd.DataFrame(cds)
cursor.close()
con.close()

日志功能

python 复制代码
import logging
import os

if os.path.exists('log_retry.log'):
    os.remove('log_retry.log')

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S',
                    filename='log_retry.log',
                    filemode='w')
count = 0
try:
    logging.info('############# TOTAL number ############:{}, '.format(count))
except:
    logging.error()

kafka传输

python 复制代码
from kafka import KafkaProducer
import json
# pip install kafka-python==2.0.2 -i https://pypi.tuna.tsinghua.edu.cn/simple
def kfk_send(msg):
    # kafka
    kafka_topic = '' #库名
    kafka_bootstrap_servers = ['172.25.214.75:9092', '172.25.214.76:9092', '172.25.214.78:9092']
    producer = KafkaProducer(bootstrap_servers=kafka_bootstrap_servers,
                             value_serializer=lambda v: json.dumps(v).encode('utf-8'))
    producer.send(kafka_topic, value=msg)
    producer.flush()

head = []
value = []
ndata = dict(zip(head, value))
kfk_send(ndata)

flask服务,异步执行(服务及时返回+耗时任务),使用线程池

python 复制代码
from concurrent.futures import ThreadPoolExecutor
from flask import Flask, request
import json
from time import sleep

executor = ThreadPoolExecutor(max_workers=4)
app = Flask(__name__)

def task(name):
    print(f"Hello {name}")
    
@app.route("/", methods=["POST"])
def main():
    request_dict = json.loads(request.data)
    p1 = request_dict["p1"]
    p2 = request_dict["p2"]
    executor.submit(task, p1, p2) #ubmit(fn, *args, **kwargs)
    sleep(3)
    return "Get your POST!!!"

if __name__ == '__main__':
    app.run()

python多进程

python 复制代码
from multiprocessing import Process

def infer(i, filelist):
    print(i, filelist)

if __name__ == '__main__':
    img_list = []
    num_process = 5
    num = int(len(img_list) / num_process)

    process_list = []
    for i in range(num_process):
        filelist = img_list[i * num:(i + 1) * num]
        if i == num_process - 1:
            filelist = img_list[i * num:]
        process_list.append(Process(target=infer, args=(i, filelist)))
    [p.start() for p in process_list]
    [p.join() for p in process_list]
相关推荐
2401_8685347840 分钟前
逻辑网络设计
网络·python
2601_967979121 小时前
python如何组织文件?python文件操作全介绍!
python·文件操作·压缩文件·遍历文件夹·shutil模块
青少儿编程课堂1 小时前
蓝桥杯·全国青少年信息素养大赛·电子学会Python/C++考级:扫描线与线段树求矩形面积并解析
c++·python·算法·bfs·信息学竞赛
利威尔xu1 小时前
Go channel 关闭 3 个 panic:向已关闭 channel 发送,服务当场崩溃
开发语言·后端·golang·并发编程·channel
羲云网络1 小时前
GPT Image 2.6 商品图交付:用文件指纹排查错图、覆盖与版本串线
图像处理·人工智能·python·技术教程·文件校验
山楂树の1 小时前
Java BigDecimal笔记
java·开发语言
wuyk5552 小时前
Spider-第14讲-表单提交与CSRF
前端·爬虫·python·elasticsearch·csrf
汤姆宝库2 小时前
RoPE旋转位置编码的实现与长上下文外推
python·深度学习·机器学习
用户019027581613 小时前
股票代码格式对照:各数据源代码转 AlphaFeed
python
__underline__3 小时前
冷门Python知识点005篇:list.sort( )的排序稳定性,是用0字节额外内存实现的
python·算法