SQLite管理定时任务执行记录的最佳实践
定时任务跑完就忘，这是很多量化脚本的通病。任务失败、重复执行、耗时异常，全靠人工盯日志，效率太低。用SQLite做任务执行记录，轻量、零配置、单文件，非常适合个人量化项目的任务管理。
本文分享一套基于SQLite的任务执行记录方案，包含表结构设计、写入、查询、清理和重试管理，代码可直接复用。
为什么要用SQLite管理任务记录
- 零依赖 ：Python标准库自带
sqlite3，无需安装额外数据库 - 单文件 ：一个
.db文件搞定，备份迁移方便 - 并发安全：SQLite支持多进程读写，配合WAL模式效果更好
- 结构化查询：比翻日志文件高效得多
数据库表设计
核心表设计如下：
sql
CREATE TABLE IF NOT EXISTS task_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
task_name TEXT NOT NULL,
start_time TEXT NOT NULL,
end_time TEXT,
status TEXT NOT NULL CHECK(status IN ('running', 'success', 'failed', 'retry')),
exit_code INTEGER,
retry_count INTEGER DEFAULT 0,
error_msg TEXT,
log_path TEXT,
created_at TEXT DEFAULT (datetime('now', 'localtime'))
);
CREATE INDEX IF NOT EXISTS idx_task_name ON task_runs(task_name);
CREATE INDEX IF NOT EXISTS idx_status ON task_runs(status);
CREATE INDEX IF NOT EXISTS idx_start_time ON task_runs(start_time);字段说明：
task_name：任务唯一标识，如daily_data_updatestart_time/end_time：任务起止时间，格式YYYY-MM-DD HH:MM:SSstatus：任务状态，running表示执行中，success成功，failed失败，retry重试中exit_code：进程退出码，0为正常retry_count：当前重试次数error_msg：错误信息摘要，方便快速定位log_path：完整日志文件路径，需要详细排查时查看
核心操作封装
用一个类封装所有数据库操作，方便在多个任务中复用。
python
import sqlite3
import json
from datetime import datetime
from pathlib import Path
class TaskDB:
def __init__(self, db_path: str = "tasks.db"):
self.db_path = db_path
self._init_db()
def _get_conn(self):
conn = sqlite3.connect(self.db_path)
conn.row_factory = sqlite3.Row
# 启用WAL模式，提高并发读写性能
conn.execute("PRAGMA journal_mode=WAL")
return conn
def _init_db(self):
with self._get_conn() as conn:
conn.execute("""
CREATE TABLE IF NOT EXISTS task_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
task_name TEXT NOT NULL,
start_time TEXT NOT NULL,
end_time TEXT,
status TEXT NOT NULL,
exit_code INTEGER,
retry_count INTEGER DEFAULT 0,
error_msg TEXT,
log_path TEXT,
created_at TEXT DEFAULT (datetime('now', 'localtime'))
)
""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_task_name ON task_runs(task_name)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_status ON task_runs(status)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_start_time ON task_runs(start_time)")
def start_task(self, task_name: str, log_path: str = "") -> int:
"""记录任务开始，返回run_id"""
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with self._get_conn() as conn:
cur = conn.execute(
"INSERT INTO task_runs (task_name, start_time, status, log_path) VALUES (?, ?, 'running', ?)",
(task_name, now, log_path)
)
return cur.lastrowid
def finish_task(self, run_id: int, status: str, exit_code: int = 0, error_msg: str = ""):
"""记录任务结束"""
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with self._get_conn() as conn:
conn.execute(
"UPDATE task_runs SET end_time=?, status=?, exit_code=?, error_msg=? WHERE id=?",
(now, status, exit_code, error_msg, run_id)
)
def mark_retry(self, run_id: int, error_msg: str = ""):
"""标记任务进入重试状态"""
with self._get_conn() as conn:
conn.execute(
"UPDATE task_runs SET status='retry', error_msg=? WHERE id=?",
(error_msg, run_id)
)
def get_recent_runs(self, task_name: str = "", limit: int = 20) -> list:
"""查询最近的任务执行记录"""
sql = "SELECT * FROM task_runs"
params = []
if task_name:
sql += " WHERE task_name = ?"
params.append(task_name)
sql += " ORDER BY start_time DESC LIMIT ?"
params.append(limit)
with self._get_conn() as conn:
rows = conn.execute(sql, params).fetchall()
return [dict(row) for row in rows]
def get_task_stats(self, task_name: str, days: int = 7) -> dict:
"""统计任务最近N天的执行情况"""
with self._get_conn() as conn:
row = conn.execute("""
SELECT
COUNT(*) as total_runs,
SUM(CASE WHEN status='success' THEN 1 ELSE 0 END) as success_count,
SUM(CASE WHEN status='failed' THEN 1 ELSE 0 END) as failed_count,
AVG(CASE WHEN status='success'
THEN (julianday(end_time) - julianday(start_time)) * 86400
ELSE NULL END) as avg_duration
FROM task_runs
WHERE task_name = ?
AND start_time >= datetime('now', ?)
""", (task_name, f'-{days} days')).fetchone()
return {
'total_runs': row['total_runs'] or 0,
'success_count': row['success_count'] or 0,
'failed_count': row['failed_count'] or 0,
'avg_duration_sec': round(row['avg_duration'], 2) if row['avg_duration'] else 0
}定时任务中的实际用法
结合schedule库演示一个完整流程：
python
import schedule
import time
import subprocess
import sys
from pathlib import Path
# 初始化数据库
db = TaskDB("task_records.db")
def run_quant_task(task_name: str, script_path: str, max_retries: int = 3):
"""
执行量化任务并记录状态
:param task_name: 任务名称
:param script_path: 要执行的Python脚本路径
:param max_retries: 最大重试次数
"""
# 记录任务开始
log_dir = Path("logs")
log_dir.mkdir(exist_ok=True)
log_file = log_dir / f"{task_name}_{time.strftime('%Y%m%d_%H%M%S')}.log"
run_id = db.start_task(task_name, str(log_file))
retry_count = 0
while retry_count <= max_retries:
try:
# 执行子进程任务
with open(log_file, 'w') as f:
result = subprocess.run(
[sys.executable, script_path],
capture_output=True,
text=True,
timeout=3600 # 1小时超时
)
f.write(result.stdout)
if result.stderr:
f.write(f"\n[STDERR]\n{result.stderr}")
if result.returncode == 0:
db.finish_task(run_id, 'success', exit_code=0)
print(f"[{task_name}] 执行成功")
return True
else:
error_msg = f"exit_code: {result.returncode}, stderr: {result.stderr[:500]}"
retry_count += 1
if retry_count <= max_retries:
db.mark_retry(run_id, error_msg)
print(f"[{task_name}] 失败，第{retry_count}次重试")
time.sleep(30 * retry_count) # 递增等待时间
else:
db.finish_task(run_id, 'failed', exit_code=result.returncode, error_msg=error_msg)
print(f"[{task_name}] 重试{max_retries}次仍失败")
return False
except subprocess.TimeoutExpired:
error_msg = "任务执行超时(>1h)"
retry_count += 1
if retry_count <= max_retries:
db.mark_retry(run_id, error_msg)
print(f"[{task_name}] 超时，第{retry_count}次重试")
time.sleep(60)
else:
db.finish_task(run_id, 'failed', exit_code=-1, error_msg=error_msg)
print(f"[{task_name}] 超时且重试完毕")
return False
except Exception as e:
error_msg = f"未知异常: {str(e)[:500]}"
db.finish_task(run_id, 'failed', exit_code=-1, error_msg=error_msg)
print(f"[{task_name}] 异常: {error_msg}")
return False
return False
# 定义定时任务
def daily_data_update_job():
run_quant_task("daily_data_update", "scripts/update_data.py")
def weekly_strategy_backtest_job():
run_quant_task("weekly_backtest", "scripts/backtest.py")
if __name__ == "__main__":
# 每天早上8点更新数据
schedule.every().day.at("08:00").do(daily_data_update_job)
# 每周一早上9点跑回测
schedule.every().monday.at("09:00").do(weekly_strategy_backtest_job)
print("定时任务已启动...")
while True:
schedule.run_pending()
time.sleep(60) # 每分钟检查一次定期清理过期记录
任务记录会不断累积，需要定期清理。保留最近90天数据，同时清理超过30天的retry状态记录：
python
def cleanup_old_records(db_path: str = "tasks.db", days: int = 90):
"""清理过期任务记录"""
db = TaskDB(db_path)
with db._get_conn() as conn:
# 删除超过90天的记录
cur = conn.execute("""
DELETE FROM task_runs
WHERE start_time < datetime('now', ?)
""", (f'-{days} days',))
print(f"清理了{cur.rowcount}条过期记录")
# 清理超过30天仍处于retry状态的记录
cur = conn.execute("""
DELETE FROM task_runs
WHERE status = 'retry'
AND start_time < datetime('now', '-30 days')
""")
print(f"清理了{cur.rowcount}条陈旧重试记录")
# 可选：清理running状态但超过24小时的记录（可能是异常中断）
cur = conn.execute("""
UPDATE task_runs
SET status = 'failed', error_msg = '任务中断，超时未完成'
WHERE status = 'running'
AND start_time < datetime('now', '-24 hours')
""")
print(f"修复了{cur.rowcount}条异常中断记录")
# 在任务启动时调用
if __name__ == "__main__":
cleanup_old_records()
# ... 其他启动逻辑查询与可视化统计
python
def generate_task_report(db_path: str = "tasks.db"):
"""生成任务执行报告"""
db = TaskDB(db_path)
print("=" * 60)
print("定时任务执行报告")
print("=" * 60)
# 获取所有任务名称
with db._get_conn() as conn:
tasks = conn.execute("SELECT DISTINCT task_name FROM task_runs").fetchall()
for task in tasks:
task_name = task['task_name']
stats_7d = db.get_task_stats(task_name, days=7)
stats_30d = db.get_task_stats(task_name, days=30)
print(f"\n任务: {task_name}")
print(f" 近7天: 总执行{stats_7d['total_runs']}次, "
f"成功{stats_7d['success_count']}次, "
f"失败{stats_7d['failed_count']}次, "
f"平均耗时{stats_7d['avg_duration_sec']}秒")
print(f" 近30天: 总执行{stats_30d['total_runs']}次, "
f"成功{stats_30d['success_count']}次, "
f"失败{stats_30d['failed_count']}次")
# 显示最近3次执行记录
recent = db.get_recent_runs(task_name, limit=3)
for run in recent:
print(f" [{run['start_time']}] {run['status']} "
f"耗时{(run.get('end_time') or '')} "
f"错误:{run.get('error_msg') or '无'}")
print("\n" + "=" * 60)
# 使用示例
if __name__ == "__main__":
generate_task_report()注意事项
- 时区问题 ：示例中统一使用
localtime，如果服务器是UTC时间，建议统一改为UTC，避免混淆 - 日志文件管理 ：
log_path字段只存路径，日志文件本身需要另外的清理机制，可以按天压缩归档 - 并发写入：如果多个进程同时写数据库，务必开启WAL模式（代码中已包含）
- 索引优化 ：如果数据量超过10万条，建议增加
(task_name, start_time)联合索引
扩展建议
- 结合
APScheduler替代schedule库，支持更复杂的调度策略 - 增加
webhook通知，任务失败时发送到钉钉或企业微信 - 将统计结果输出为HTML报告，方便查看
这套方案已经在我自己的量化项目中稳定运行了大半年，管理着十几个定时任务，排查问题基本不需要翻日志文件，直接查数据库就能定位。
更多技术实战内容，欢迎继续关注本站。