SQLAlchemy 复杂 SQL 执行指南与模板库

一、核心执行方式

1. text() 原生 SQL 执行（最常用）

from sqlalchemy import text, create_engine

engine = create_engine("postgresql://user:pass@localhost/db")

# 基础模板
with engine.connect() as conn:
    result = conn.execute(
        text("""
            SELECT u.id, u.name, COUNT(o.id) as order_count
            FROM users u
            LEFT JOIN orders o ON u.id = o.user_id
            WHERE u.created_at > :start_date
            GROUP BY u.id, u.name
            HAVING COUNT(o.id) > :min_orders
            ORDER BY order_count DESC
            LIMIT :limit
        """),
        {
            "start_date": "2024-01-01",
            "min_orders": 5,
            "limit": 10
        }
    )
    rows = result.fetchall()

2. Connection + 事务管理

from sqlalchemy import text
from sqlalchemy.engine import Connection

def complex_operation(conn: Connection):
    # 开启事务
    with conn.begin():
        # 批量更新
        conn.execute(
            text("""
                UPDATE products p
                SET stock = stock - :quantity
                FROM (
                    SELECT product_id, SUM(quantity) as quantity
                    FROM order_items
                    WHERE order_id = :order_id
                    GROUP BY product_id
                ) oi
                WHERE p.id = oi.product_id
            """),
            {"order_id": 123}
        )
        
        # 插入日志
        conn.execute(
            text("""
                INSERT INTO inventory_logs 
                (product_id, change_type, quantity_change, created_by)
                SELECT product_id, 'SALE', -SUM(quantity), :user_id
                FROM order_items
                WHERE order_id = :order_id
                GROUP BY product_id
            """),
            {"order_id": 123, "user_id": 456}
        )

# 使用
with engine.connect() as conn:
    complex_operation(conn)

二、复杂 SQL 模板库

模板 1：窗口函数分析

def window_function_analysis():
    sql = text("""
        WITH monthly_stats AS (
            SELECT 
                DATE_TRUNC('month', order_date) as month,
                category_id,
                SUM(amount) as total_sales,
                ROW_NUMBER() OVER (PARTITION BY category_id ORDER BY SUM(amount) DESC) as rank_in_category,
                LAG(SUM(amount)) OVER (PARTITION BY category_id ORDER BY DATE_TRUNC('month', order_date)) as prev_month_sales
            FROM orders o
            JOIN order_items oi ON o.id = oi.order_id
            JOIN products p ON oi.product_id = p.id
            WHERE o.order_date >= :start_date
            GROUP BY DATE_TRUNC('month', order_date), category_id
        )
        SELECT 
            month,
            category_id,
            total_sales,
            ROUND((total_sales - prev_month_sales) / prev_month_sales * 100, 2) as growth_rate_percent
        FROM monthly_stats
        WHERE rank_in_category <= 3
        ORDER BY month, total_sales DESC
    """)
    
    with engine.connect() as conn:
        return conn.execute(sql, {"start_date": "2024-01-01"}).fetchall()

模板 2：递归 CTE（树形结构）

def recursive_cte_department_hierarchy():
    sql = text("""
        WITH RECURSIVE dept_tree AS (
            -- 锚定成员
            SELECT 
                id, 
                name, 
                parent_id, 
                0 as level,
                ARRAY[id] as path
            FROM departments
            WHERE parent_id IS NULL
            
            UNION ALL
            
            -- 递归成员
            SELECT 
                d.id,
                d.name,
                d.parent_id,
                dt.level + 1,
                dt.path || d.id
            FROM departments d
            INNER JOIN dept_tree dt ON d.parent_id = dt.id
            WHERE NOT d.id = ANY(dt.path)  -- 防止循环
        )
        SELECT 
            id,
            name,
            parent_id,
            level,
            array_to_string(path, ' -> ') as hierarchy_path
        FROM dept_tree
        ORDER BY path
    """)
    
    with engine.connect() as conn:
        return conn.execute(sql).fetchall()

模板 3：动态条件构建（防 SQL 注入）

from sqlalchemy import text
from typing import Dict, Any, List

class DynamicQueryBuilder:
    def __init__(self):
        self.params = {}
        self.conditions = []
        
    def add_condition(self, field: str, value: Any, operator: str = "="):
        if value is None:
            return self
            
        param_name = f"param_{len(self.params)}"
        self.params[param_name] = value
        self.conditions.append(f"{field} {operator} :{param_name}")
        return self
    
    def build(self, base_query: str) -> tuple[str, Dict]:
        where_clause = ""
        if self.conditions:
            where_clause = "WHERE " + " AND ".join(self.conditions)
        
        final_sql = base_query.replace("{where}", where_clause)
        return final_sql, self.params

# 使用示例
builder = DynamicQueryBuilder()
builder.add_condition("age", 18, ">=")
builder.add_condition("status", "active")
builder.add_condition("created_at", "2024-01-01", ">=")

base_query = """
    SELECT * FROM users
    {where}
    ORDER BY created_at DESC
"""

sql, params = builder.build(base_query)

with engine.connect() as conn:
    result = conn.execute(text(sql), params)

模板 4：批量数据操作（UPSERT）

def bulk_upsert_users(users_data: List[Dict]):
    """PostgreSQL 的 UPSERT 示例"""
    sql = text("""
        INSERT INTO users (id, email, name, updated_at)
        VALUES (:id, :email, :name, NOW())
        ON CONFLICT (email) 
        DO UPDATE SET
            name = EXCLUDED.name,
            updated_at = EXCLUDED.updated_at,
            version = users.version + 1
        RETURNING id, email, name, version
    """)
    
    results = []
    with engine.begin() as conn:  # begin() 自动提交/回滚
        for user in users_data:
            result = conn.execute(sql, user)
            results.extend(result.fetchall())
    return results

模板 5：JSON/JSONB 复杂查询

def jsonb_complex_query():
    sql = text("""
        SELECT 
            id,
            data->>'title' as title,
            jsonb_array_elements(data->'tags') as tag,
            jsonb_path_query_first(data, '$.metadata.rating') as rating
        FROM articles
        WHERE 
            data @> '{"status": "published"}'
            AND data->'metadata'->>'category' = :category
            AND jsonb_array_length(data->'comments') > :min_comments
            AND data->'metadata'->'rating' ?| ARRAY['good', 'excellent']
        ORDER BY (data->'metadata'->>'views')::INTEGER DESC
    """)
    
    with engine.connect() as conn:
        return conn.execute(
            sql, 
            {"category": "tech", "min_comments": 5}
        ).fetchall()

模板 6：性能优化 - 分页与游标

def efficient_pagination(last_seen_id: int = 0, batch_size: int = 1000):
    """基于游标的分页（比 OFFSET 高效）"""
    sql = text("""
        SELECT id, data
        FROM large_table
        WHERE id > :last_seen_id
        ORDER BY id
        LIMIT :batch_size
    """)
    
    with engine.connect() as conn:
        while True:
            rows = conn.execute(
                sql, 
                {"last_seen_id": last_seen_id, "batch_size": batch_size}
            ).fetchall()
            
            if not rows:
                break
                
            yield rows
            last_seen_id = rows[-1].id

模板 7：临时表处理复杂逻辑

def process_with_temp_table():
    sql = text("""
        -- 创建临时表
        CREATE TEMP TABLE temp_order_summary ON COMMIT DROP AS
        SELECT 
            user_id,
            COUNT(*) as order_count,
            SUM(total_amount) as total_spent,
            MAX(order_date) as last_order_date
        FROM orders
        WHERE order_date >= CURRENT_DATE - INTERVAL '1 year'
        GROUP BY user_id;
        
        -- 使用临时表进行复杂分析
        SELECT 
            u.id,
            u.email,
            t.order_count,
            t.total_spent,
            CASE 
                WHEN t.total_spent > 10000 THEN 'VIP'
                WHEN t.total_spent > 5000 THEN 'Gold'
                ELSE 'Regular'
            END as customer_tier
        FROM users u
        JOIN temp_order_summary t ON u.id = t.user_id
        WHERE t.last_order_date >= CURRENT_DATE - INTERVAL '30 days'
        ORDER BY t.total_spent DESC;
    """)
    
    with engine.begin() as conn:
        return conn.execute(sql).fetchall()

模板 8：多数据库兼容写法

from sqlalchemy import text
from sqlalchemy.engine import Engine

def cross_db_compatible_query(engine: Engine):
    dialect = engine.dialect.name
    
    if dialect == 'postgresql':
        date_func = "DATE_TRUNC('day', created_at)"
        limit_clause = "LIMIT :limit"
    elif dialect == 'mysql':
        date_func = "DATE_FORMAT(created_at, '%Y-%m-%d')"
        limit_clause = "LIMIT :limit"
    elif dialect == 'sqlite':
        date_func = "DATE(created_at)"
        limit_clause = "LIMIT :limit"
    else:
        raise ValueError(f"Unsupported dialect: {dialect}")
    
    sql = text(f"""
        SELECT 
            {date_func} as day,
            COUNT(*) as count
        FROM events
        WHERE event_type = :event_type
        GROUP BY {date_func}
        ORDER BY day DESC
        {limit_clause}
    """)
    
    with engine.connect() as conn:
        return conn.execute(
            sql, 
            {"event_type": "login", "limit": 30}
        ).fetchall()

三、高级技巧与最佳实践

1. 结果映射为字典

from sqlalchemy import text

def fetch_as_dict():
    sql = text("""
        SELECT id, name, email 
        FROM users 
        WHERE status = :status
    """)
    
    with engine.connect() as conn:
        result = conn.execute(sql, {"status": "active"})
        # 方法1：使用 result.mappings()
        return [dict(row) for row in result.mappings()]
        
        # 方法2：手动映射
        columns = result.keys()
        return [dict(zip(columns, row)) for row in result.fetchall()]

2. 流式处理结果（大数据量）

def stream_large_result():
    sql = text("""
        SELECT * FROM large_table
        WHERE created_at > :start_date
        ORDER BY id
    """)
    
    with engine.connect().execution_options(stream_results=True) as conn:
        result = conn.execution_options(yield_per=1000).execute(
            sql, {"start_date": "2024-01-01"}
        )
        
        for partition in result.partitions():
            for row in partition:
                process_row(row)

3. SQL 注入防护检查

import re
from sqlalchemy import exc

def safe_execute(query_str: str, params: dict):
    # 简单检查：禁止字符串拼接
    dangerous_patterns = [
        r'\'\s*\|\|',  # 字符串拼接
        r';\s*DROP',   # 多重语句
        r'--\s*$',     # 注释攻击
    ]
    
    for pattern in dangerous_patterns:
        if re.search(pattern, query_str, re.IGNORECASE):
            raise ValueError(f"Potentially unsafe SQL detected: {pattern}")
    
    try:
        with engine.connect() as conn:
            return conn.execute(text(query_str), params)
    except exc.SQLAlchemyError as e:
        # 记录日志但不暴露详细信息
        logger.error(f"SQL execution error: {type(e).__name__}")
        raise

4. 调试与日志

import logging
from sqlalchemy import event

# 启用 SQL 日志
logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)

# 监听事件获取实际执行的 SQL
@event.listens_for(engine, "before_cursor_execute")
def before_cursor_execute(conn, cursor, statement, parameters, context, executemany):
    logger.debug(f"Executing SQL: {statement}")
    logger.debug(f"Parameters: {parameters}")

# 获取编译后的 SQL（用于调试）
def get_compiled_sql(statement, bind=None):
    """适用于 Core/ORM 混合场景"""
    from sqlalchemy.sql import compiler
    if bind is None:
        bind = engine
    
    dialect = bind.dialect
    comp = compiler.SQLCompiler(dialect, statement)
    comp.compile()
    return str(comp)

四、ORM 与 Core 混合模式

from sqlalchemy.orm import Session
from sqlalchemy import select, func
from models import User, Order

def hybrid_approach():
    with Session(engine) as session:
        # ORM 部分
        subq = (
            select(
                Order.user_id,
                func.count(Order.id).label('order_count')
            )
            .group_by(Order.user_id)
            .subquery()
        )
        
        # 混合原生 SQL
        stmt = text("""
            SELECT 
                u.id,
                u.name,
                COALESCE(s.order_count, 0) as order_count,
                CASE 
                    WHEN COALESCE(s.order_count, 0) > 10 THEN 'VIP'
                    ELSE 'Regular'
                END as tier
            FROM users u
            LEFT JOIN ({subq}) s ON u.id = s.user_id
            WHERE u.status = 'active'
        """).bindparams(subq=subq.selectable)
        
        result = session.execute(stmt)
        return result.fetchall()

五、错误处理与重试机制

from sqlalchemy import exc
from tenacity import retry, stop_after_attempt, wait_exponential

@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=4, max=10),
    retry=lambda e: isinstance(e, (exc.OperationalError, exc.DBAPIError))
)
def execute_with_retry(sql: str, params: dict):
    with engine.connect() as conn:
        try:
            return conn.execute(text(sql), params)
        except exc.IntegrityError as e:
            # 唯一约束冲突等，不需要重试
            logger.warning(f"Integrity error: {e}")
            raise
        except Exception as e:
            logger.error(f"Unexpected error: {e}")
            raise

六、总结建议

1. 优先使用 text() ：对于复杂 SQL，直接使用 text() 比构建 Core/ORM 表达式更直观
2. 始终参数化 ：永远使用 :param 语法传递参数，避免 SQL 注入
3. 事务管理 ：复杂操作务必放在事务中（with conn.begin()）
4. 性能考虑 ：
   • 大数据量使用流式处理
   • 避免 OFFSET 分页，使用游标分页
   • 合理使用临时表和 CTE
5. 可维护性 ：
   • 将复杂 SQL 组织为函数或类
   • 添加充分的注释
   • 考虑多数据库兼容性