消息队列对比
特性 Celery + Redis RabbitMQ Apache Kafka
消息模型 任务队列 消息代理 事件流平台
持久化 可选 支持 强持久化
消息顺序 不保证 队列内保证 分区内保证
吞吐量 中等 中等 非常高
延迟 低 低 低到中等
适用场景 异步任务、定时任务 应用解耦、复杂路由 事件流、数据管道
复杂度 低 中等 高
数据重放 不支持 有限支持 完整支持
生态系统 Python生态 多语言支持 完整流生态
业务场景：电商订单处理系统
消息队列需求：
- 订单创建后的异步处理（库存扣减、积分计算、通知发送）
- 支付结果异步通知
- 物流状态更新
- 用户行为追踪
- 数据同步和ETL处理
- 延迟任务（订单超时取消）
Celery + Redis 实现
text
celery_redis_project/
├── app.py
├── config.py
├── tasks/
│ ├── __init__.py
│ ├── order_tasks.py
│ ├── payment_tasks.py
│ ├── notification_tasks.py
│ └── user_tasks.py
├── models/
│ └── order.py
├── workers/
│ ├── __init__.py
│ ├── order_worker.py
│ └── notification_worker.py
└── requirements.txt
python
import os
from datetime import timedelta
class Config:
REDIS_URL = os.getenv("REDIS_URL","redis://localhost:6379/0")
#Celery配置
CELERY_BROKER_URL = REDIS_URL
CELERY_RESULT_BACKEND = REDIS_URL
CELERY_TIMEZONE = "Asia/Shanghai"
CELERY_ENABLE_UTC = True
# 任务路由
CELERY_TASK_ROUTES = {
'tasks.order_tasks.*': {'queue': 'orders'},
'tasks.payment_tasks.*': {'queue': 'payments'},
'tasks.notification_tasks.*': {'queue': 'notifications'},
'tasks.user_tasks.*': {'queue': 'users'},
}
# 定时任务
CELERY_BEAT_SCHEDULE = {
'check-order-timeout': {
'task': 'tasks.order_tasks.check_order_timeout',
'schedule': timedelta(minutes=1), # 每分钟检查一次
},
'clean-expired-carts': {
'task': 'tasks.order_tasks.clean_expired_carts',
'schedule': timedelta(hours=1), # 每小时清理一次
},
'send-daily-stats': {
'task': 'tasks.notification_tasks.send_daily_stats',
'schedule': timedelta(days=1), # 每天发送统计
},
}
# 并发配置
CELERY_WORKER_CONCURRENCY = 4
CELERY_WORKER_PREFETCH_MULTIPLIER = 1
CELERY_TASK_ACKS_LATE = True # 任务执行完成后确认
CELERY_TASK_REJECT_ON_WORKER_LOST = True
# 重试配置
CELERY_TASK_DEFAULT_RETRY_DELAY = 30 # 30秒后重试
CELERY_TASK_MAX_RETRIES = 3
config = Config()
python
from celery import Celery
from config import Config
def create_celery_app():
"""创建Celery应用"""
app = Celery('ecommerce')
#配置Celery
app.config_from_objecct(Config)
#自动发现任务
app.autodiscover_tasks([
'tasks.order_tasks',
'tasks.payment_tasks',
'tasks.notification_tasks',
'tasks.user_tasks'
])
return app
#创建全局Celery实例
celery_app = create_celery_app()
if __name__ == '__main__':
celery_app.start()tasks/order_tasks.py
python
import time
import logging
from datetime import datetime,timedelta
from celery.exceptions import MaxRetriesExceededError
from app import celery_app
from models.order import Order,OrderItem,Inventory
logger = logging.getLogger(__name__)
@celery_app.task(bind=True, max_retries=3, default_retry_delay=30)
def process_order_creation(self, order_id):
"""
处理订单创建后的异步任务
- 库存扣减
- 积分计算
- 订单状态更新
"""
try:
logger.info(f"开始处理订单 {order_id}")
#模拟获取订单数据
order = get_order_by_id(order_id)
if not order:
logger.error(f"订单 {order_id} 不存在")
return False
# 1.扣减库存
if not debuct_inventory():
logger.error(f"订单 {order_id} 库存扣减失败")
raise Exception("库存扣减失败")
# 2.计算积分
calculate_points.delay(order_id,order['user_id'],order['total_amount'])
# 3.发送订单创建通知
send_order_created_notification.delay(order_id,order['user_id'])
# 4.更新订单状态为确认
update_order_status(order_id,'confirmed')
logger.info(f"订单 {order_id} 处理完成")
return True
except Exception as exc:
logger.error(f"订单处理失败: {exc}")
try:
#重试任务
raise self.retry(exc=exc,countdown=60)
except MaxRetiresExceededError:
#重试次数用尽，标记订单为失败
update_order_status(order_id,'failed')
logger.error(f"订单 {order_id} 处理失败，已达到最大重试次数")
return False
@celery_app.task(bind=True)
def deduct_inventory(self, order):
"""扣减库存"""
try:
for item in order['items']]:
#遍历每个订单条目
product_id = item['product_id']
quantity = item['quantity']
#模拟库存扣减
logger.info(f"扣减商品 {product_id} 库存 {quantity}")
time.sleep(0.1) # 模拟数据库操作
# 这里应该是实际的库存扣减逻辑
# inventory = Inventory.get(product_id=product_id)
# if inventory.quantity < quantity:
# raise Exception(f"商品 {product_id} 库存不足")
# inventory.quantity -= quantity
# inventory.save()
return True
except Exception as exc:
logger.error(f"库存扣减失败: {exc}")
raise exc
@celery_app.task
def calculate_points(order_id, user_id, amount):
"""计算用户积分"""
try:
points = int(amount) # 1元=1积分
logger.info(f"为用户 {user_id} 计算积分 {points}")
# 模拟积分计算
time.sleep(0.2)
# 这里应该是实际的积分计算逻辑
# user = User.get(user_id)
# user.points += points
# user.save()
# 发送积分变动通知
send_points_notification.delay(user_id, points, 'order_created')
return True
except Exception as exc:
logger.error(f"积分计算失败: {exc}")
return False
@celery_app.task(bind=True, max_retries=5)
def check_order_timeout(self):
"""检查超时未支付订单"""
try:
logger.info("开始检查超时订单")
# 获取超时订单（创建时间超过30分钟未支付）
timeout_threshold = datetime.now() - timedelta(minutes=30)
# timeout_orders = Order.query.filter(
# Order.status == 'pending',
# Order.created_at < timeout_threshold
# ).all()
# 模拟超时订单
timeout_orders = [1, 2, 3] # 模拟订单ID
for order_id in timeout_orders:
cancel_unpaid_order.delay(order_id)
logger.info(f"找到 {len(timeout_orders)} 个超时订单")
return len(timeout_orders)
except Exception as exc:
logger.error(f"检查超时订单失败: {exc}")
raise self.retry(exc=exc)
@celery_app.task
def cancel_unpaid_order(order_id):
"""取消未支付订单"""
try:
logger.info(f"取消未支付订单 {order_id}")
# 1. 恢复库存
# order = Order.get(order_id)
# for item in order.items:
# inventory = Inventory.get(product_id=item.product_id)
# inventory.quantity += item.quantity
# inventory.save()
# 2. 更新订单状态
# order.status = 'cancelled'
# order.cancelled_at = datetime.now()
# order.save()
# 3. 发送取消通知
send_order_cancelled_notification.delay(order_id)
logger.info(f"订单 {order_id} 已取消")
return True
except Exception as exc:
logger.error(f"取消订单失败: {exc}")
return False
@celery_app.task
def clean_expired_carts():
"""清理过期购物车"""
try:
logger.info("开始清理过期购物车")
# 清理24小时未更新的购物车
# expired_time = datetime.now() - timedelta(hours=24)
# expired_carts = Cart.query.filter(
# Cart.updated_at < expired_time
# ).delete()
expired_carts = 10 # 模拟清理数量
logger.info(f"清理了 {expired_carts} 个过期购物车")
return expired_carts
except Exception as exc:
logger.error(f"清理购物车失败: {exc}")
return 0
# 模拟函数
def get_order_by_id(order_id):
"""模拟获取订单"""
return {
'id': order_id,
'user_id': 1,
'total_amount': 199.99,
'status': 'pending',
'items': [
{'product_id': 1, 'quantity': 2, 'price': 99.99},
{'product_id': 2, 'quantity': 1, 'price': 50.00}
]
}
def update_order_status(order_id, status):
"""模拟更新订单状态"""
logger.info(f"更新订单 {order_id} 状态为 {status}")
return True tasks/payment_tasks.py
python
import logging
from datetime import datetime
from app import celery_app
logger = logging.getLogger(__name__)
@celery_app.task(bind=True,max_retries=3)
def process_payment_callback(self, payment_id, status, amount):
"""处理支付回调"""
try:
logger.info(f"处理支付回调: {payment_id}, 状态: {status}, 金额: {amount}")
# 1. 更新支付状态
update_payment_status(payment_id, status)
if status == 'success':
# 支付成功处理
handle_successful_payment.delay(payment_id)
elif status == 'failed':
# 支付失败处理
handle_failed_payment.delay(payment_id)
elif status == 'refunded':
# 退款处理
handle_refund.delay(payment_id)
return True
except Exception as exc:
logger.error(f"支付回调处理失败: {exc}")
raise self.retry(exc=exc)
@celery_app.task
def handle_successful_payment(payment_id):
"""处理成功支付"""
try:
logger.info(f"处理成功支付: {payment_id}")
# 1. 获取关联订单
order_id = get_order_by_payment(payment_id)
# 2. 更新订单状态为已支付
update_order_status(order_id, 'paid')
update_order_paid_time(order_id, datetime.now())
# 3. 触发后续流程
from .order_tasks import process_order_creation
process_order_creation.delay(order_id)
# 4. 发送支付成功通知
from .notification_tasks import send_payment_success_notification
send_payment_success_notification.delay(payment_id, order_id)
logger.info(f"支付 {payment_id} 处理完成")
return True
except Exception as exc:
logger.error(f"成功支付处理失败: {exc}")
return False
@celery_app.task
def handle_failed_payment(payment_id):
"""处理失败支付"""
try:
logger.info(f"处理失败支付: {payment_id}")
# 1. 获取关联订单
order_id = get_order_by_payment(payment_id)
# 2. 更新订单状态
update_order_status(order_id, 'payment_failed')
# 3. 发送支付失败通知
from .notification_tasks import send_payment_failed_notification
send_payment_failed_notification.delay(payment_id, order_id)
logger.info(f"失败支付 {payment_id} 处理完成")
return True
except Exception as exc:
logger.error(f"失败支付处理失败: {exc}")
return False
@celery_app.task
def handle_refund(payment_id):
"""处理退款"""
try:
logger.info(f"处理退款: {payment_id}")
# 1. 获取退款详情
refund_amount = get_refund_amount(payment_id)
# 2. 执行退款逻辑
execute_refund(payment_id, refund_amount)
# 3. 更新相关状态
update_refund_status(payment_id, 'completed')
# 4. 发送退款通知
from .notification_tasks import send_refund_notification
send_refund_notification.delay(payment_id, refund_amount)
logger.info(f"退款 {payment_id} 处理完成")
return True
except Exception as exc:
logger.error(f"退款处理失败: {exc}")
return False
# 模拟函数
def update_payment_status(payment_id, status):
logger.info(f"更新支付 {payment_id} 状态为 {status}")
return True
def get_order_by_payment(payment_id):
return 1 # 模拟订单ID
def update_order_paid_time(order_id, paid_time):
logger.info(f"更新订单 {order_id} 支付时间为 {paid_time}")
return True
def get_refund_amount(payment_id):
return 100.0 # 模拟退款金额
def execute_refund(payment_id, amount):
logger.info(f"执行退款: {payment_id}, 金额: {amount}")
return True
def update_refund_status(payment_id, status):
logger.info(f"更新退款状态: {payment_id} -> {status}")
return Truetasks/notification_tasks.py
python
import time
import logging
from datetime import datetime
from app import celery_app
logger = logging.getLogger(__name__)
@celery_app.task(bind=True, max_retries=3)
def send_order_created_notification(self, order_id, user_id):
"""发送订单创建通知"""
try:
logger.info(f"发送订单创建通知: 订单 {order_id}, 用户 {user_id}")
# 1. 准备通知数据
order_data = get_order_data(order_id)
user_data = get_user_data(user_id)
# 2. 发送邮件
if user_data.get('email'):
send_email.delay(
to_email=user_data['email'],
subject="订单创建成功",
template="order_created",
data={'order': order_data, 'user': user_data}
)
# 3. 发送短信
if user_data.get('phone'):
send_sms.delay(
to_phone=user_data['phone'],
template="order_created",
data={'order': order_data}
)
# 4. 发送App推送
send_app_push.delay(
user_id=user_id,
title="订单创建成功",
content=f"您的订单 {order_id} 已创建",
data={'order_id': order_id}
)
logger.info(f"订单创建通知发送完成: {order_id}")
return True
except Exception as exc:
logger.error(f"订单创建通知发送失败: {exc}")
raise self.retry(exc=exc)
@celery_app.task
def send_payment_success_notification(payment_id, order_id):
"""发送支付成功通知"""
logger.info(f"发送支付成功通知: {payment_id}, 订单 {order_id}")
# 模拟发送通知
time.sleep(0.1)
return True
@celery_app.task
def send_payment_failed_notification(payment_id, order_id):
"""发送支付失败通知"""
logger.info(f"发送支付失败通知: {payment_id}, 订单 {order_id}")
# 模拟发送通知
time.sleep(0.1)
return True
@celery_app.task
def send_order_cancelled_notification(order_id):
"""发送订单取消通知"""
logger.info(f"发送订单取消通知: {order_id}")
# 模拟发送通知
time.sleep(0.1)
return True
@celery_app.task
def send_refund_notification(payment_id, amount):
"""发送退款通知"""
logger.info(f"发送退款通知: {payment_id}, 金额 {amount}")
# 模拟发送通知
time.sleep(0.1)
return True
@celery_app.task
def send_points_notification(user_id, points, reason):
"""发送积分变动通知"""
logger.info(f"发送积分通知: 用户 {user_id}, 积分 {points}, 原因 {reason}")
# 模拟发送通知
time.sleep(0.1)
return True
@celery_app.task
def send_daily_stats():
"""发送每日统计"""
try:
logger.info("发送每日统计")
# 1. 生成统计报告
stats = generate_daily_stats()
# 2. 发送给管理员
send_email.delay(
to_email="admin@example.com",
subject="每日统计报告",
template="daily_stats",
data={'stats': stats}
)
logger.info("每日统计发送完成")
return True
except Exception as exc:
logger.error(f"每日统计发送失败: {exc}")
return False
@celery_app.task
def send_email(to_email, subject, template, data):
"""发送邮件任务"""
logger.info(f"发送邮件到 {to_email}: {subject}")
time.sleep(0.2) # 模拟邮件发送
return True
@celery_app.task
def send_sms(to_phone, template, data):
"""发送短信任务"""
logger.info(f"发送短信到 {to_phone}")
time.sleep(0.1) # 模拟短信发送
return True
@celery_app.task
def send_app_push(user_id, title, content, data):
"""发送App推送"""
logger.info(f"发送App推送给用户 {user_id}: {title}")
time.sleep(0.1) # 模拟推送发送
return True
# 模拟函数
def get_order_data(order_id):
return {'id': order_id, 'total_amount': 199.99}
def get_user_data(user_id):
return {'id': user_id, 'email': 'user@example.com', 'phone': '13800138000'}
def generate_daily_stats():
return {
'date': datetime.now().strftime('%Y-%m-%d'),
'new_orders': 150,
'total_revenue': 29985.50,
'active_users': 1200
}tasks/user_tasks.py
python
# tasks/user_tasks.py
import time
import logging
from datetime import datetime, timedelta
from app import celery_app
logger = logging.getLogger(__name__)
@celery_app.task
def track_user_behavior(user_id, action, data):
"""追踪用户行为"""
try:
logger.info(f"追踪用户行为: 用户 {user_id}, 动作 {action}")
# 1. 记录到分析系统
record_user_analytics(user_id, action, data)
# 2. 更新用户画像
update_user_profile(user_id, action, data)
# 3. 检查是否触发营销活动
check_marketing_campaigns.delay(user_id, action, data)
return True
except Exception as exc:
logger.error(f"用户行为追踪失败: {exc}")
return False
@celery_app.task
def check_marketing_campaigns(user_id, action, data):
"""检查营销活动"""
try:
logger.info(f"检查营销活动: 用户 {user_id}, 动作 {action}")
# 基于用户行为触发不同的营销活动
if action == 'product_view':
# 商品浏览后推荐相似商品
recommend_similar_products.delay(user_id, data.get('product_id'))
elif action == 'add_to_cart':
# 加入购物车后发送优惠券
if data.get('cart_count', 0) > 2:
send_abandoned_cart_coupon.delay(user_id)
elif action == 'purchase_completed':
# 购买完成后发送感谢邮件和推荐
send_thank_you_email.delay(user_id, data.get('order_id'))
recommend_related_products.delay(user_id, data.get('product_ids', []))
return True
except Exception as exc:
logger.error(f"营销活动检查失败: {exc}")
return False
@celery_app.task
def recommend_similar_products(user_id, product_id):
"""推荐相似商品"""
logger.info(f"为用户 {user_id} 推荐相似商品，基于商品 {product_id}")
time.sleep(0.1)
return True
@celery_app.task
def send_abandoned_cart_coupon(user_id):
"""发送购物车放弃优惠券"""
logger.info(f"为用户 {user_id} 发送购物车放弃优惠券")
time.sleep(0.1)
return True
@celery_app.task
def send_thank_you_email(user_id, order_id):
"""发送感谢邮件"""
logger.info(f"为用户 {user_id} 发送感谢邮件，订单 {order_id}")
time.sleep(0.1)
return True
@celery_app.task
def recommend_related_products(user_id, product_ids):
"""推荐相关商品"""
logger.info(f"为用户 {user_id} 推荐相关商品")
time.sleep(0.1)
return True
@celery_app.task
def sync_user_data_to_es(user_id):
"""同步用户数据到Elasticsearch"""
try:
logger.info(f"同步用户 {user_id} 数据到ES")
# 获取用户数据
user_data = get_user_data_for_es(user_id)
# 同步到ES
# es.index(index='users', id=user_id, body=user_data)
logger.info(f"用户 {user_id} 数据同步完成")
return True
except Exception as exc:
logger.error(f"用户数据同步失败: {exc}")
return False
# 模拟函数
def record_user_analytics(user_id, action, data):
logger.info(f"记录用户分析数据: {user_id}, {action}")
return True
def update_user_profile(user_id, action, data):
logger.info(f"更新用户画像: {user_id}, {action}")
return True
def get_user_data_for_es(user_id):
return {
'user_id': user_id,
'name': f'用户{user_id}',
'last_active': datetime.now().isoformat()
}workers/order_worker.py
python
# workers/order_worker.py
#!/usr/bin/env python3
"""
订单处理Worker启动脚本
"""
import os
import sys
from app import celery_app
if __name__ == '__main__':
# 启动订单队列的Worker
argv = [
'worker',
'--loglevel=info',
'--queues=orders,payments',
'--concurrency=2',
'--hostname=order_worker@%h'
]
celery_app.worker_main(argv)workers/notification_worker.py
python
# workers/notification_worker.py
#!/usr/bin/env python3
"""
通知处理Worker启动脚本
"""
import os
import sys
from app import celery_app
if __name__ == '__main__':
# 启动通知队列的Worker
argv = [
'worker',
'--loglevel=info',
'--queues=notifications',
'--concurrency=4', # 通知任务可以更多并发
'--hostname=notification_worker@%h'
]
celery_app.worker_main(argv)
python
# 使用示例 - API接口
from flask import Flask, request, jsonify
from tasks.order_tasks import process_order_creation
from tasks.payment_tasks import process_payment_callback
from tasks.user_tasks import track_user_behavior
import logging
app = Flask(__name__)
@app.route('/api/orders', methods=['POST'])
def create_order():
"""创建订单接口"""
try:
data = request.get_json()
# 1. 创建订单（同步）
order_id = create_order_in_db(data)
# 2. 异步处理订单后续任务
process_order_creation.delay(order_id)
# 3. 追踪用户行为
track_user_behavior.delay(
user_id=data['user_id'],
action='create_order',
data={'order_id': order_id, 'amount': data['total_amount']}
)
return jsonify({
'success': True,
'order_id': order_id,
'message': '订单创建成功'
})
except Exception as e:
logging.error(f"订单创建失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
@app.route('/api/payments/callback', methods=['POST'])
def payment_callback():
"""支付回调接口"""
try:
data = request.get_json()
# 异步处理支付回调
process_payment_callback.delay(
payment_id=data['payment_id'],
status=data['status'],
amount=data['amount']
)
return jsonify({'success': True})
except Exception as e:
logging.error(f"支付回调处理失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
def create_order_in_db(data):
"""模拟创建订单到数据库"""
return 12345 # 返回订单ID
if __name__ == '__main__':
app.run(debug=True)RabbitMQ 实现
text
rabbitmq_project/
├── producer/
│ ├── __init__.py
│ ├── order_producer.py
│ ├── payment_producer.py
│ └── notification_producer.py
├── consumer/
│ ├── __init__.py
│ ├── order_consumer.py
│ ├── payment_consumer.py
│ └── notification_consumer.py
├── common/
│ ├── __init__.py
│ ├── connection.py
│ └── message_schema.py
└── requirements.txtcommon/connection.py
python
import pika
import json
import logging
from typing import Any,Dict,Callable
import threading
logger = logging.getLogger(__name__)
class RabbitMQConnection:
"""RabbitMQ连接管理"""
_instance = None
_lock = threading.Lock()
def __new__(cls):
with cls._lock:
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._connections = {}
return cls._instance
def get_connection(self, connection_name='default'):
"""获取RabbitMQ连接"""
if connection_name not in self._connections:
self._create_connection(connection_name)
return self._connections[connection_name]
def _create_connection(self, connection_name):
"""创建新的连接"""
try:
# RabbitMQ连接参数
credentials = pika.PlainCredentials(
username='guest', # 生产环境使用环境变量
password='guest'
)
parameters = pika.ConnectionParameters(
host='localhost',
port=5672,
credentials=credentials,
heartbeat=600,
blocked_connection_timeout=300
)
connection = pika.BlockingConnection(parameters)
self._connections[connection_name] = connection
logger.info(f"RabbitMQ连接 {connection_name} 创建成功")
except Exception as e:
logger.error(f"RabbitMQ连接创建失败: {e}")
raise
def close_all(self):
"""关闭所有连接"""
for name, connection in self._connections.items():
try:
connection.close()
logger.info(f"RabbitMQ连接 {name} 已关闭")
except Exception as e:
logger.error(f"关闭连接 {name} 失败: {e}")
self._connections.clear()
# 全局连接管理器
connection_manager = RabbitMQConnection()
python
# common/message_schema.py
from pydantic import BaseModel, Field
from typing import Any, Dict, Optional
from datetime import datetime
from enum import Enum
class MessageType(str, Enum):
ORDER_CREATED = "order.created"
ORDER_UPDATED = "order.updated"
ORDER_CANCELLED = "order.cancelled"
PAYMENT_SUCCESS = "payment.success"
PAYMENT_FAILED = "payment.failed"
NOTIFICATION_SEND = "notification.send"
USER_BEHAVIOR = "user.behavior"
class BaseMessage(BaseModel):
"""基础消息模型"""
message_id: str = Field(..., description="消息ID")
message_type: MessageType = Field(..., description="消息类型")
timestamp: datetime = Field(default_factory=datetime.now, description="消息时间戳")
source: str = Field(..., description="消息来源")
version: str = Field(default="1.0", description="消息版本")
class Config:
use_enum_values = True
class OrderMessage(BaseMessage):
"""订单消息"""
order_id: int = Field(..., description="订单ID")
user_id: int = Field(..., description="用户ID")
total_amount: float = Field(..., description="订单总金额")
items: list = Field(default=[], description="订单商品")
metadata: Dict[str, Any] = Field(default={}, description="扩展数据")
class PaymentMessage(BaseMessage):
"""支付消息"""
payment_id: str = Field(..., description="支付ID")
order_id: int = Field(..., description="订单ID")
amount: float = Field(..., description="支付金额")
status: str = Field(..., description="支付状态")
payment_method: str = Field(..., description="支付方式")
metadata: Dict[str, Any] = Field(default={}, description="扩展数据")
class NotificationMessage(BaseMessage):
"""通知消息"""
user_id: int = Field(..., description="用户ID")
notification_type: str = Field(..., description="通知类型")
title: str = Field(..., description="通知标题")
content: str = Field(..., description="通知内容")
channels: list = Field(default=[], description="发送渠道")
metadata: Dict[str, Any] = Field(default={}, description="扩展数据")
class UserBehaviorMessage(BaseMessage):
"""用户行为消息"""
user_id: int = Field(..., description="用户ID")
action: str = Field(..., description="用户行为")
page: Optional[str] = Field(None, description="页面")
product_id: Optional[int] = Field(None, description="商品ID")
metadata: Dict[str, Any] = Field(default={}, description="扩展数据")
python
# producer/order_producer.py
import json
import logging
import uuid
from datetime import datetime
from common.connection import connection_manager
from common.message_schema import OrderMessage, MessageType
logger = logging.getLogger(__name__)
class OrderProducer:
"""订单消息生产者"""
def __init__(self):
self.connection = connection_manager.get_connection('order_producer')
self.channel = self.connection.channel()
# 声明交换机和队列
self._setup_infrastructure()
def _setup_infrastructure(self):
"""设置RabbitMQ基础设施"""
# 声明主题交换机
self.channel.exchange_declare(
exchange='order_events',
exchange_type='topic',
durable=True
)
# 声明订单相关队列
queues = [
('order_processing', 'order.created'),
('order_notifications', 'order.*'),
('order_analytics', 'order.#'),
]
for queue_name, routing_key in queues:
self.channel.queue_declare(queue=queue_name, durable=True)
self.channel.queue_bind(
exchange='order_events',
queue=queue_name,
routing_key=routing_key
)
logger.info("订单消息基础设施设置完成")
def send_order_created(self, order_data: dict):
"""发送订单创建消息"""
try:
message = OrderMessage(
message_id=str(uuid.uuid4()),
message_type=MessageType.ORDER_CREATED,
source="order_service",
order_id=order_data['id'],
user_id=order_data['user_id'],
total_amount=order_data['total_amount'],
items=order_data.get('items', []),
metadata=order_data.get('metadata', {})
)
self._publish_message(
exchange='order_events',
routing_key='order.created',
message=message
)
logger.info(f"订单创建消息发送成功: {order_data['id']}")
return True
except Exception as e:
logger.error(f"订单创建消息发送失败: {e}")
return False
def send_order_updated(self, order_data: dict):
"""发送订单更新消息"""
try:
message = OrderMessage(
message_id=str(uuid.uuid4()),
message_type=MessageType.ORDER_UPDATED,
source="order_service",
order_id=order_data['id'],
user_id=order_data['user_id'],
total_amount=order_data['total_amount'],
metadata=order_data.get('metadata', {})
)
self._publish_message(
exchange='order_events',
routing_key='order.updated',
message=message
)
logger.info(f"订单更新消息发送成功: {order_data['id']}")
return True
except Exception as e:
logger.error(f"订单更新消息发送失败: {e}")
return False
def send_order_cancelled(self, order_data: dict):
"""发送订单取消消息"""
try:
message = OrderMessage(
message_id=str(uuid.uuid4()),
message_type=MessageType.ORDER_CANCELLED,
source="order_service",
order_id=order_data['id'],
user_id=order_data['user_id'],
total_amount=order_data['total_amount'],
metadata={'reason': order_data.get('cancel_reason', 'unknown')}
)
self._publish_message(
exchange='order_events',
routing_key='order.cancelled',
message=message
)
logger.info(f"订单取消消息发送成功: {order_data['id']}")
return True
except Exception as e:
logger.error(f"订单取消消息发送失败: {e}")
return False
def _publish_message(self, exchange: str, routing_key: str, message: OrderMessage):
"""发布消息到RabbitMQ"""
self.channel.basic_publish(
exchange=exchange,
routing_key=routing_key,
body=message.json(),
properties=pika.BasicProperties(
delivery_mode=2, # 持久化消息
content_type='application/json',
timestamp=int(datetime.now().timestamp())
)
)
def close(self):
"""关闭连接"""
if self.channel and not self.channel.is_closed:
self.channel.close()
logger.info("订单生产者连接已关闭")
python
# producer/payment_producer.py
import json
import logging
import uuid
from datetime import datetime
from common.connection import connection_manager
from common.message_schema import PaymentMessage, MessageType
logger = logging.getLogger(__name__)
class PaymentProducer:
"""支付消息生产者"""
def __init__(self):
self.connection = connection_manager.get_connection('payment_producer')
self.channel = self.connection.channel()
# 声明交换机和队列
self._setup_infrastructure()
def _setup_infrastructure(self):
"""设置RabbitMQ基础设施"""
# 声明直连交换机
self.channel.exchange_declare(
exchange='payment_events',
exchange_type='direct',
durable=True
)
# 声明支付相关队列
queues = [
('payment_processing', 'payment.success'),
('payment_processing', 'payment.failed'),
('payment_notifications', 'payment.success'),
('payment_notifications', 'payment.failed'),
('payment_analytics', 'payment.success'),
('payment_analytics', 'payment.failed'),
]
for queue_name, routing_key in queues:
self.channel.queue_declare(queue=queue_name, durable=True)
self.channel.queue_bind(
exchange='payment_events',
queue=queue_name,
routing_key=routing_key
)
logger.info("支付消息基础设施设置完成")
def send_payment_success(self, payment_data: dict):
"""发送支付成功消息"""
try:
message = PaymentMessage(
message_id=str(uuid.uuid4()),
message_type=MessageType.PAYMENT_SUCCESS,
source="payment_service",
payment_id=payment_data['id'],
order_id=payment_data['order_id'],
amount=payment_data['amount'],
status='success',
payment_method=payment_data.get('method', 'unknown'),
metadata=payment_data.get('metadata', {})
)
self._publish_message(
exchange='payment_events',
routing_key='payment.success',
message=message
)
logger.info(f"支付成功消息发送成功: {payment_data['id']}")
return True
except Exception as e:
logger.error(f"支付成功消息发送失败: {e}")
return False
def send_payment_failed(self, payment_data: dict):
"""发送支付失败消息"""
try:
message = PaymentMessage(
message_id=str(uuid.uuid4()),
message_type=MessageType.PAYMENT_FAILED,
source="payment_service",
payment_id=payment_data['id'],
order_id=payment_data['order_id'],
amount=payment_data['amount'],
status='failed',
payment_method=payment_data.get('method', 'unknown'),
metadata={
'error_code': payment_data.get('error_code'),
'error_message': payment_data.get('error_message'),
**payment_data.get('metadata', {})
}
)
self._publish_message(
exchange='payment_events',
routing_key='payment.failed',
message=message
)
logger.info(f"支付失败消息发送成功: {payment_data['id']}")
return True
except Exception as e:
logger.error(f"支付失败消息发送失败: {e}")
return False
def _publish_message(self, exchange: str, routing_key: str, message: PaymentMessage):
"""发布消息到RabbitMQ"""
self.channel.basic_publish(
exchange=exchange,
routing_key=routing_key,
body=message.json(),
properties=pika.BasicProperties(
delivery_mode=2, # 持久化消息
content_type='application/json',
timestamp=int(datetime.now().timestamp())
)
)
def close(self):
"""关闭连接"""
if self.channel and not self.channel.is_closed:
self.channel.close()
logger.info("支付生产者连接已关闭")
python
# consumer/order_consumer.py
import json
import logging
import time
from common.connection import connection_manager
from common.message_schema import OrderMessage, PaymentMessage
logger = logging.getLogger(__name__)
class OrderConsumer:
"""订单消息消费者"""
def __init__(self):
self.connection = connection_manager.get_connection('order_consumer')
self.channel = self.connection.channel()
# 配置QoS
self.channel.basic_qos(prefetch_count=1)
def start_consuming(self):
"""开始消费消息"""
try:
# 消费订单处理队列
self.channel.basic_consume(
queue='order_processing',
on_message_callback=self._process_order_message,
auto_ack=False
)
# 消费支付处理队列（用于订单状态更新）
self.channel.basic_consume(
queue='payment_processing',
on_message_callback=self._process_payment_message,
auto_ack=False
)
logger.info("订单消费者开始监听消息...")
self.channel.start_consuming()
except Exception as e:
logger.error(f"订单消费者启动失败: {e}")
raise
def _process_order_message(self, ch, method, properties, body):
"""处理订单消息"""
try:
message_data = json.loads(body)
message = OrderMessage(**message_data)
logger.info(f"处理订单消息: {message.message_type}, 订单ID: {message.order_id}")
if message.message_type == "order.created":
self._handle_order_created(message)
elif message.message_type == "order.updated":
self._handle_order_updated(message)
elif message.message_type == "order.cancelled":
self._handle_order_cancelled(message)
# 确认消息
ch.basic_ack(delivery_tag=method.delivery_tag)
logger.info(f"订单消息处理完成: {message.message_id}")
except Exception as e:
logger.error(f"订单消息处理失败: {e}")
# 拒绝消息并重新入队
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
def _process_payment_message(self, ch, method, properties, body):
"""处理支付消息"""
try:
message_data = json.loads(body)
message = PaymentMessage(**message_data)
logger.info(f"处理支付消息: {message.message_type}, 支付ID: {message.payment_id}")
if message.message_type == "payment.success":
self._handle_payment_success(message)
elif message.message_type == "payment.failed":
self._handle_payment_failed(message)
# 确认消息
ch.basic_ack(delivery_tag=method.delivery_tag)
logger.info(f"支付消息处理完成: {message.message_id}")
except Exception as e:
logger.error(f"支付消息处理失败: {e}")
# 拒绝消息并重新入队
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
def _handle_order_created(self, message: OrderMessage):
"""处理订单创建"""
logger.info(f"处理订单创建: {message.order_id}")
# 1. 扣减库存
self._deduct_inventory(message.order_id, message.items)
# 2. 计算积分
self._calculate_points(message.user_id, message.total_amount)
# 3. 更新订单状态
self._update_order_status(message.order_id, 'confirmed')
logger.info(f"订单创建处理完成: {message.order_id}")
def _handle_order_updated(self, message: OrderMessage):
"""处理订单更新"""
logger.info(f"处理订单更新: {message.order_id}")
# 订单更新逻辑
time.sleep(0.1)
def _handle_order_cancelled(self, message: OrderMessage):
"""处理订单取消"""
logger.info(f"处理订单取消: {message.order_id}")
# 1. 恢复库存
self._restore_inventory(message.order_id, message.items)
# 2. 更新订单状态
self._update_order_status(message.order_id, 'cancelled')
logger.info(f"订单取消处理完成: {message.order_id}")
def _handle_payment_success(self, message: PaymentMessage):
"""处理支付成功"""
logger.info(f"处理支付成功: {message.payment_id}, 订单: {message.order_id}")
# 1. 更新订单支付状态
self._update_order_payment_status(message.order_id, 'paid')
# 2. 触发订单处理
self._handle_order_created_after_payment(message.order_id)
logger.info(f"支付成功处理完成: {message.payment_id}")
def _handle_payment_failed(self, message: PaymentMessage):
"""处理支付失败"""
logger.info(f"处理支付失败: {message.payment_id}, 订单: {message.order_id}")
# 更新订单支付状态
self._update_order_payment_status(message.order_id, 'payment_failed')
logger.info(f"支付失败处理完成: {message.payment_id}")
def _deduct_inventory(self, order_id, items):
"""扣减库存"""
logger.info(f"为订单 {order_id} 扣减库存")
for item in items:
logger.info(f"扣减商品 {item.get('product_id')} 库存 {item.get('quantity')}")
time.sleep(0.05)
def _restore_inventory(self, order_id, items):
"""恢复库存"""
logger.info(f"为订单 {order_id} 恢复库存")
for item in items:
logger.info(f"恢复商品 {item.get('product_id')} 库存 {item.get('quantity')}")
time.sleep(0.05)
def _calculate_points(self, user_id, amount):
"""计算积分"""
points = int(amount)
logger.info(f"为用户 {user_id} 计算积分 {points}")
time.sleep(0.1)
def _update_order_status(self, order_id, status):
"""更新订单状态"""
logger.info(f"更新订单 {order_id} 状态为 {status}")
time.sleep(0.05)
def _update_order_payment_status(self, order_id, status):
"""更新订单支付状态"""
logger.info(f"更新订单 {order_id} 支付状态为 {status}")
time.sleep(0.05)
def _handle_order_created_after_payment(self, order_id):
"""支付后处理订单创建"""
logger.info(f"支付后处理订单创建: {order_id}")
# 这里可以执行支付后的特定逻辑
def stop_consuming(self):
"""停止消费"""
if self.channel and self.channel.is_consuming():
self.channel.stop_consuming()
logger.info("订单消费者已停止")
if __name__ == '__main__':
consumer = OrderConsumer()
try:
consumer.start_consuming()
except KeyboardInterrupt:
consumer.stop_consuming()
finally:
connection_manager.close_all()
python
# consumer/notification_consumer.py
import json
import logging
import time
from common.connection import connection_manager
from common.message_schema import OrderMessage, PaymentMessage, NotificationMessage
logger = logging.getLogger(__name__)
class NotificationConsumer:
"""通知消息消费者"""
def __init__(self):
self.connection = connection_manager.get_connection('notification_consumer')
self.channel = self.connection.channel()
# 配置QoS
self.channel.basic_qos(prefetch_count=2) # 通知任务可以更高并发
def start_consuming(self):
"""开始消费消息"""
try:
# 消费订单通知队列
self.channel.basic_consume(
queue='order_notifications',
on_message_callback=self._process_order_notification,
auto_ack=False
)
# 消费支付通知队列
self.channel.basic_consume(
queue='payment_notifications',
on_message_callback=self._process_payment_notification,
auto_ack=False
)
logger.info("通知消费者开始监听消息...")
self.channel.start_consuming()
except Exception as e:
logger.error(f"通知消费者启动失败: {e}")
raise
def _process_order_notification(self, ch, method, properties, body):
"""处理订单通知"""
try:
message_data = json.loads(body)
if 'order_id' in message_data:
message = OrderMessage(**message_data)
logger.info(f"处理订单通知: {message.message_type}, 订单ID: {message.order_id}")
if message.message_type == "order.created":
self._send_order_created_notification(message)
elif message.message_type == "order.cancelled":
self._send_order_cancelled_notification(message)
# 确认消息
ch.basic_ack(delivery_tag=method.delivery_tag)
logger.info(f"订单通知处理完成: {message.message_id}")
except Exception as e:
logger.error(f"订单通知处理失败: {e}")
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
def _process_payment_notification(self, ch, method, properties, body):
"""处理支付通知"""
try:
message_data = json.loads(body)
message = PaymentMessage(**message_data)
logger.info(f"处理支付通知: {message.message_type}, 支付ID: {message.payment_id}")
if message.message_type == "payment.success":
self._send_payment_success_notification(message)
elif message.message_type == "payment.failed":
self._send_payment_failed_notification(message)
# 确认消息
ch.basic_ack(delivery_tag=method.delivery_tag)
logger.info(f"支付通知处理完成: {message.message_id}")
except Exception as e:
logger.error(f"支付通知处理失败: {e}")
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
def _send_order_created_notification(self, message: OrderMessage):
"""发送订单创建通知"""
logger.info(f"发送订单创建通知: 订单 {message.order_id}, 用户 {message.user_id}")
# 发送邮件
self._send_email(
to_email=get_user_email(message.user_id),
subject="订单创建成功",
content=f"您的订单 {message.order_id} 已创建，金额: {message.total_amount}"
)
# 发送短信
self._send_sms(
to_phone=get_user_phone(message.user_id),
content=f"订单创建成功，订单号: {message.order_id}"
)
time.sleep(0.2) # 模拟通知发送
def _send_order_cancelled_notification(self, message: OrderMessage):
"""发送订单取消通知"""
logger.info(f"发送订单取消通知: 订单 {message.order_id}")
reason = message.metadata.get('reason', '未知原因')
self._send_email(
to_email=get_user_email(message.user_id),
subject="订单已取消",
content=f"您的订单 {message.order_id} 已取消，原因: {reason}"
)
time.sleep(0.1)
def _send_payment_success_notification(self, message: PaymentMessage):
"""发送支付成功通知"""
logger.info(f"发送支付成功通知: 支付 {message.payment_id}")
self._send_email(
to_email=get_user_email_by_order(message.order_id),
subject="支付成功",
content=f"您的订单 {message.order_id} 支付成功，金额: {message.amount}"
)
time.sleep(0.1)
def _send_payment_failed_notification(self, message: PaymentMessage):
"""发送支付失败通知"""
logger.info(f"发送支付失败通知: 支付 {message.payment_id}")
error_msg = message.metadata.get('error_message', '未知错误')
self._send_email(
to_email=get_user_email_by_order(message.order_id),
subject="支付失败",
content=f"您的订单 {message.order_id} 支付失败，错误: {error_msg}"
)
time.sleep(0.1)
def _send_email(self, to_email: str, subject: str, content: str):
"""发送邮件"""
logger.info(f"发送邮件到 {to_email}: {subject}")
# 实际邮件发送逻辑
time.sleep(0.1)
def _send_sms(self, to_phone: str, content: str):
"""发送短信"""
logger.info(f"发送短信到 {to_phone}")
# 实际短信发送逻辑
time.sleep(0.05)
def stop_consuming(self):
"""停止消费"""
if self.channel and self.channel.is_consuming():
self.channel.stop_consuming()
logger.info("通知消费者已停止")
# 模拟函数
def get_user_email(user_id):
return f"user{user_id}@example.com"
def get_user_phone(user_id):
return "13800138000"
def get_user_email_by_order(order_id):
return f"user{order_id}@example.com"
if __name__ == '__main__':
consumer = NotificationConsumer()
try:
consumer.start_consuming()
except KeyboardInterrupt:
consumer.stop_consuming()
finally:
connection_manager.close_all()
python
# 使用示例 - API接口
from flask import Flask, request, jsonify
from producer.order_producer import OrderProducer
from producer.payment_producer import PaymentProducer
import logging
app = Flask(__name__)
# 创建生产者实例
order_producer = OrderProducer()
payment_producer = PaymentProducer()
@app.route('/api/orders', methods=['POST'])
def create_order():
"""创建订单接口"""
try:
data = request.get_json()
# 1. 创建订单（同步）
order_id = create_order_in_db(data)
order_data = {
'id': order_id,
'user_id': data['user_id'],
'total_amount': data['total_amount'],
'items': data.get('items', []),
'metadata': data.get('metadata', {})
}
# 2. 发送订单创建消息
order_producer.send_order_created(order_data)
return jsonify({
'success': True,
'order_id': order_id,
'message': '订单创建成功'
})
except Exception as e:
logging.error(f"订单创建失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
@app.route('/api/payments/callback', methods=['POST'])
def payment_callback():
"""支付回调接口"""
try:
data = request.get_json()
payment_data = {
'id': data['payment_id'],
'order_id': data['order_id'],
'amount': data['amount'],
'method': data.get('method', 'alipay'),
'metadata': data.get('metadata', {})
}
if data['status'] == 'success':
payment_producer.send_payment_success(payment_data)
else:
payment_data.update({
'error_code': data.get('error_code'),
'error_message': data.get('error_message')
})
payment_producer.send_payment_failed(payment_data)
return jsonify({'success': True})
except Exception as e:
logging.error(f"支付回调处理失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
@app.route('/api/orders/<int:order_id>/cancel', methods=['POST'])
def cancel_order(order_id):
"""取消订单接口"""
try:
data = request.get_json()
order_data = {
'id': order_id,
'user_id': data['user_id'],
'total_amount': 0, # 实际应该从数据库获取
'cancel_reason': data.get('reason', '用户取消')
}
# 发送订单取消消息
order_producer.send_order_cancelled(order_data)
return jsonify({
'success': True,
'message': '订单取消请求已受理'
})
except Exception as e:
logging.error(f"订单取消失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
def create_order_in_db(data):
"""模拟创建订单到数据库"""
return 12345 # 返回订单ID
@app.teardown_appcontext
def close_producers(exception=None):
"""关闭生产者连接"""
order_producer.close()
payment_producer.close()
if __name__ == '__main__':
app.run(debug=True)Apache Kafka 实现
text
kafka_project/
├── producer/
│ ├── __init__.py
│ ├── kafka_producer.py
│ └── event_producer.py
├── consumer/
│ ├── __init__.py
│ ├── order_consumer.py
│ ├── analytics_consumer.py
│ └── notification_consumer.py
├── schemas/
│ ├── __init__.py
│ └── events.py
└── requirements.txt
python
# schemas/events.py
from pydantic import BaseModel, Field
from typing import Any, Dict, List, Optional
from datetime import datetime
from enum import Enum
import json
class EventType(str, Enum):
ORDER_CREATED = "order_created"
ORDER_UPDATED = "order_updated"
ORDER_CANCELLED = "order_cancelled"
PAYMENT_COMPLETED = "payment_completed"
PAYMENT_FAILED = "payment_failed"
USER_REGISTERED = "user_registered"
USER_LOGGED_IN = "user_logged_in"
PRODUCT_VIEWED = "product_viewed"
class BaseEvent(BaseModel):
"""基础事件模型"""
event_id: str = Field(..., description="事件ID")
event_type: EventType = Field(..., description="事件类型")
event_time: datetime = Field(default_factory=datetime.now, description="事件时间")
source: str = Field(..., description="事件来源")
version: str = Field(default="1.0.0", description="事件版本")
correlation_id: Optional[str] = Field(None, description="关联ID")
class Config:
use_enum_values = True
json_encoders = {
datetime: lambda v: v.isoformat()
}
class OrderEvent(BaseEvent):
"""订单事件"""
order_id: int = Field(..., description="订单ID")
user_id: int = Field(..., description="用户ID")
total_amount: float = Field(..., description="订单金额")
currency: str = Field(default="CNY", description="货币")
items: List[Dict[str, Any]] = Field(default=[], description="订单项")
shipping_address: Optional[Dict[str, Any]] = Field(None, description="配送地址")
metadata: Dict[str, Any] = Field(default={}, description="元数据")
class PaymentEvent(BaseEvent):
"""支付事件"""
payment_id: str = Field(..., description="支付ID")
order_id: int = Field(..., description="订单ID")
amount: float = Field(..., description="支付金额")
currency: str = Field(default="CNY", description="货币")
payment_method: str = Field(..., description="支付方式")
status: str = Field(..., description="支付状态")
gateway_response: Optional[Dict[str, Any]] = Field(None, description="支付网关响应")
metadata: Dict[str, Any] = Field(default={}, description="元数据")
class UserEvent(BaseEvent):
"""用户事件"""
user_id: int = Field(..., description="用户ID")
session_id: Optional[str] = Field(None, description="会话ID")
user_agent: Optional[str] = Field(None, description="用户代理")
ip_address: Optional[str] = Field(None, description="IP地址")
device_info: Optional[Dict[str, Any]] = Field(None, description="设备信息")
metadata: Dict[str, Any] = Field(default={}, description="元数据")
class ProductEvent(BaseEvent):
"""商品事件"""
product_id: int = Field(..., description="商品ID")
user_id: Optional[int] = Field(None, description="用户ID")
category_id: Optional[int] = Field(None, description="分类ID")
price: Optional[float] = Field(None, description="价格")
action: str = Field(..., description="操作类型")
metadata: Dict[str, Any] = Field(default={}, description="元数据")
python
# producer/kafka_producer.py
import json
import logging
import uuid
from datetime import datetime
from confluent_kafka import Producer, KafkaError
from typing import Dict, Any, Optional
logger = logging.getLogger(__name__)
class KafkaEventProducer:
"""Kafka事件生产者"""
def __init__(self, bootstrap_servers: str = 'localhost:9092'):
self.bootstrap_servers = bootstrap_servers
self.producer = self._create_producer()
# 主题映射
self.topic_mapping = {
'order_events': ['order_created', 'order_updated', 'order_cancelled'],
'payment_events': ['payment_completed', 'payment_failed'],
'user_events': ['user_registered', 'user_logged_in'],
'product_events': ['product_viewed'],
}
def _create_producer(self) -> Producer:
"""创建Kafka生产者"""
config = {
'bootstrap.servers': self.bootstrap_servers,
'client.id': 'ecommerce-producer',
'acks': 'all', # 等待所有副本确认
'retries': 5, # 重试次数
'retry.backoff.ms': 1000, # 重试间隔
'compression.type': 'snappy', # 压缩类型
'batch.num.messages': 1000, # 批量消息数量
'linger.ms': 10, # 批量发送延迟
'queue.buffering.max.messages': 100000, # 队列大小
'queue.buffering.max.kbytes': 1048576, # 队列大小（KB）
}
return Producer(config)
def get_topic_for_event(self, event_type: str) -> str:
"""根据事件类型获取对应的Kafka主题"""
for topic, events in self.topic_mapping.items():
if event_type in events:
return topic
return 'default_events' # 默认主题
def produce_event(self, event_data: Dict[str, Any]) -> bool:
"""生产事件到Kafka"""
try:
event_type = event_data.get('event_type')
if not event_type:
logger.error("事件类型不能为空")
return False
# 获取目标主题
topic = self.get_topic_for_event(event_type)
# 确保事件有ID和时间戳
if 'event_id' not in event_data:
event_data['event_id'] = str(uuid.uuid4())
if 'event_time' not in event_data:
event_data['event_time'] = datetime.now().isoformat()
# 序列化消息
message_value = json.dumps(event_data, ensure_ascii=False).encode('utf-8')
# 发送消息
self.producer.produce(
topic=topic,
value=message_value,
key=event_data['event_id'].encode('utf-8'), # 使用事件ID作为Key
callback=self._delivery_callback
)
# 立即刷新（生产环境应该批量处理）
self.producer.poll(0)
logger.info(f"事件发送成功: {event_data['event_id']} -> {topic}")
return True
except Exception as e:
logger.error(f"事件发送失败: {e}")
return False
def _delivery_callback(self, err, msg):
"""消息发送回调"""
if err:
logger.error(f"消息发送失败: {err}")
else:
logger.debug(f"消息发送成功: {msg.topic()} [{msg.partition()}] @ {msg.offset()}")
def flush(self, timeout: float = 5.0):
"""刷新生产者，确保所有消息都已发送"""
self.producer.flush(timeout)
def close(self):
"""关闭生产者"""
self.flush()
logger.info("Kafka生产者已关闭")
class EventProducer:
"""事件生产者（业务层封装）"""
def __init__(self, kafka_producer: KafkaEventProducer):
self.producer = kafka_producer
def send_order_created(self, order_data: Dict[str, Any]) -> bool:
"""发送订单创建事件"""
event = {
'event_type': 'order_created',
'source': 'order_service',
'order_id': order_data['id'],
'user_id': order_data['user_id'],
'total_amount': order_data['total_amount'],
'items': order_data.get('items', []),
'metadata': order_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_order_updated(self, order_data: Dict[str, Any]) -> bool:
"""发送订单更新事件"""
event = {
'event_type': 'order_updated',
'source': 'order_service',
'order_id': order_data['id'],
'user_id': order_data['user_id'],
'total_amount': order_data['total_amount'],
'metadata': order_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_order_cancelled(self, order_data: Dict[str, Any]) -> bool:
"""发送订单取消事件"""
event = {
'event_type': 'order_cancelled',
'source': 'order_service',
'order_id': order_data['id'],
'user_id': order_data['user_id'],
'metadata': {
'reason': order_data.get('cancel_reason', 'unknown'),
**order_data.get('metadata', {})
}
}
return self.producer.produce_event(event)
def send_payment_completed(self, payment_data: Dict[str, Any]) -> bool:
"""发送支付完成事件"""
event = {
'event_type': 'payment_completed',
'source': 'payment_service',
'payment_id': payment_data['id'],
'order_id': payment_data['order_id'],
'amount': payment_data['amount'],
'payment_method': payment_data.get('method', 'unknown'),
'status': 'completed',
'gateway_response': payment_data.get('gateway_response'),
'metadata': payment_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_payment_failed(self, payment_data: Dict[str, Any]) -> bool:
"""发送支付失败事件"""
event = {
'event_type': 'payment_failed',
'source': 'payment_service',
'payment_id': payment_data['id'],
'order_id': payment_data['order_id'],
'amount': payment_data['amount'],
'payment_method': payment_data.get('method', 'unknown'),
'status': 'failed',
'metadata': {
'error_code': payment_data.get('error_code'),
'error_message': payment_data.get('error_message'),
**payment_data.get('metadata', {})
}
}
return self.producer.produce_event(event)
def send_user_registered(self, user_data: Dict[str, Any]) -> bool:
"""发送用户注册事件"""
event = {
'event_type': 'user_registered',
'source': 'user_service',
'user_id': user_data['id'],
'metadata': user_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_product_viewed(self, product_data: Dict[str, Any]) -> bool:
"""发送商品浏览事件"""
event = {
'event_type': 'product_viewed',
'source': 'web_frontend',
'product_id': product_data['id'],
'user_id': product_data.get('user_id'),
'category_id': product_data.get('category_id'),
'price': product_data.get('price'),
'action': 'view',
'metadata': product_data.get('metadata', {})
}
return self.producer.produce_event(event)
python
# producer/kafka_producer.py
import json
import logging
import uuid
from datetime import datetime
from confluent_kafka import Producer, KafkaError
from typing import Dict, Any, Optional
logger = logging.getLogger(__name__)
class KafkaEventProducer:
"""Kafka事件生产者"""
def __init__(self, bootstrap_servers: str = 'localhost:9092'):
self.bootstrap_servers = bootstrap_servers
self.producer = self._create_producer()
# 主题映射
self.topic_mapping = {
'order_events': ['order_created', 'order_updated', 'order_cancelled'],
'payment_events': ['payment_completed', 'payment_failed'],
'user_events': ['user_registered', 'user_logged_in'],
'product_events': ['product_viewed'],
}
def _create_producer(self) -> Producer:
"""创建Kafka生产者"""
config = {
'bootstrap.servers': self.bootstrap_servers,
'client.id': 'ecommerce-producer',
'acks': 'all', # 等待所有副本确认
'retries': 5, # 重试次数
'retry.backoff.ms': 1000, # 重试间隔
'compression.type': 'snappy', # 压缩类型
'batch.num.messages': 1000, # 批量消息数量
'linger.ms': 10, # 批量发送延迟
'queue.buffering.max.messages': 100000, # 队列大小
'queue.buffering.max.kbytes': 1048576, # 队列大小（KB）
}
return Producer(config)
def get_topic_for_event(self, event_type: str) -> str:
"""根据事件类型获取对应的Kafka主题"""
for topic, events in self.topic_mapping.items():
if event_type in events:
return topic
return 'default_events' # 默认主题
def produce_event(self, event_data: Dict[str, Any]) -> bool:
"""生产事件到Kafka"""
try:
event_type = event_data.get('event_type')
if not event_type:
logger.error("事件类型不能为空")
return False
# 获取目标主题
topic = self.get_topic_for_event(event_type)
# 确保事件有ID和时间戳
if 'event_id' not in event_data:
event_data['event_id'] = str(uuid.uuid4())
if 'event_time' not in event_data:
event_data['event_time'] = datetime.now().isoformat()
# 序列化消息
message_value = json.dumps(event_data, ensure_ascii=False).encode('utf-8')
# 发送消息
self.producer.produce(
topic=topic,
value=message_value,
key=event_data['event_id'].encode('utf-8'), # 使用事件ID作为Key
callback=self._delivery_callback
)
# 立即刷新（生产环境应该批量处理）
self.producer.poll(0)
logger.info(f"事件发送成功: {event_data['event_id']} -> {topic}")
return True
except Exception as e:
logger.error(f"事件发送失败: {e}")
return False
def _delivery_callback(self, err, msg):
"""消息发送回调"""
if err:
logger.error(f"消息发送失败: {err}")
else:
logger.debug(f"消息发送成功: {msg.topic()} [{msg.partition()}] @ {msg.offset()}")
def flush(self, timeout: float = 5.0):
"""刷新生产者，确保所有消息都已发送"""
self.producer.flush(timeout)
def close(self):
"""关闭生产者"""
self.flush()
logger.info("Kafka生产者已关闭")
class EventProducer:
"""事件生产者（业务层封装）"""
def __init__(self, kafka_producer: KafkaEventProducer):
self.producer = kafka_producer
def send_order_created(self, order_data: Dict[str, Any]) -> bool:
"""发送订单创建事件"""
event = {
'event_type': 'order_created',
'source': 'order_service',
'order_id': order_data['id'],
'user_id': order_data['user_id'],
'total_amount': order_data['total_amount'],
'items': order_data.get('items', []),
'metadata': order_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_order_updated(self, order_data: Dict[str, Any]) -> bool:
"""发送订单更新事件"""
event = {
'event_type': 'order_updated',
'source': 'order_service',
'order_id': order_data['id'],
'user_id': order_data['user_id'],
'total_amount': order_data['total_amount'],
'metadata': order_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_order_cancelled(self, order_data: Dict[str, Any]) -> bool:
"""发送订单取消事件"""
event = {
'event_type': 'order_cancelled',
'source': 'order_service',
'order_id': order_data['id'],
'user_id': order_data['user_id'],
'metadata': {
'reason': order_data.get('cancel_reason', 'unknown'),
**order_data.get('metadata', {})
}
}
return self.producer.produce_event(event)
def send_payment_completed(self, payment_data: Dict[str, Any]) -> bool:
"""发送支付完成事件"""
event = {
'event_type': 'payment_completed',
'source': 'payment_service',
'payment_id': payment_data['id'],
'order_id': payment_data['order_id'],
'amount': payment_data['amount'],
'payment_method': payment_data.get('method', 'unknown'),
'status': 'completed',
'gateway_response': payment_data.get('gateway_response'),
'metadata': payment_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_payment_failed(self, payment_data: Dict[str, Any]) -> bool:
"""发送支付失败事件"""
event = {
'event_type': 'payment_failed',
'source': 'payment_service',
'payment_id': payment_data['id'],
'order_id': payment_data['order_id'],
'amount': payment_data['amount'],
'payment_method': payment_data.get('method', 'unknown'),
'status': 'failed',
'metadata': {
'error_code': payment_data.get('error_code'),
'error_message': payment_data.get('error_message'),
**payment_data.get('metadata', {})
}
}
return self.producer.produce_event(event)
def send_user_registered(self, user_data: Dict[str, Any]) -> bool:
"""发送用户注册事件"""
event = {
'event_type': 'user_registered',
'source': 'user_service',
'user_id': user_data['id'],
'metadata': user_data.get('metadata', {})
}
return self.producer.produce_event(event)
def send_product_viewed(self, product_data: Dict[str, Any]) -> bool:
"""发送商品浏览事件"""
event = {
'event_type': 'product_viewed',
'source': 'web_frontend',
'product_id': product_data['id'],
'user_id': product_data.get('user_id'),
'category_id': product_data.get('category_id'),
'price': product_data.get('price'),
'action': 'view',
'metadata': product_data.get('metadata', {})
}
return self.producer.produce_event(event)
python
# consumer/order_consumer.py
import json
import logging
import time
from typing import Dict, Any
from confluent_kafka import Consumer, KafkaError, KafkaException
from schemas.events import OrderEvent, PaymentEvent
logger = logging.getLogger(__name__)
class OrderEventConsumer:
"""订单事件消费者"""
def __init__(self, bootstrap_servers: str = 'localhost:9092', group_id: str = 'order-processor'):
self.bootstrap_servers = bootstrap_servers
self.group_id = group_id
self.consumer = self._create_consumer()
self.running = False
# 订阅主题
self.topics = ['order_events', 'payment_events']
def _create_consumer(self) -> Consumer:
"""创建Kafka消费者"""
config = {
'bootstrap.servers': self.bootstrap_servers,
'group.id': self.group_id,
'auto.offset.reset': 'earliest', # 从最早开始消费
'enable.auto.commit': False, # 手动提交偏移量
'max.poll.interval.ms': 300000, # 最大poll间隔
'session.timeout.ms': 10000, # 会话超时
'heartbeat.interval.ms': 3000, # 心跳间隔
}
consumer = Consumer(config)
consumer.subscribe(self.topics)
return consumer
def start_consuming(self):
"""开始消费事件"""
self.running = True
logger.info("订单事件消费者开始监听...")
try:
while self.running:
msg = self.consumer.poll(1.0) # 1秒超时
if msg is None:
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
# 分区结束，正常情况
continue
else:
logger.error(f"消费者错误: {msg.error()}")
continue
try:
# 处理消息
self._process_message(msg)
# 手动提交偏移量
self.consumer.commit(asynchronous=False)
except Exception as e:
logger.error(f"消息处理失败: {e}")
# 可以选择将失败消息发送到死信队列
self._send_to_dlq(msg, str(e))
except KeyboardInterrupt:
logger.info("收到中断信号，停止消费...")
except Exception as e:
logger.error(f"消费者异常: {e}")
finally:
self.stop_consuming()
def _process_message(self, msg):
"""处理单个消息"""
try:
# 反序列化消息
event_data = json.loads(msg.value().decode('utf-8'))
topic = msg.topic()
logger.info(f"处理事件: {event_data.get('event_type')} from {topic}")
if topic == 'order_events':
self._handle_order_event(event_data)
elif topic == 'payment_events':
self._handle_payment_event(event_data)
except json.JSONDecodeError as e:
logger.error(f"消息JSON解析失败: {e}")
except Exception as e:
logger.error(f"消息处理异常: {e}")
raise
def _handle_order_event(self, event_data: Dict[str, Any]):
"""处理订单事件"""
event = OrderEvent(**event_data)
if event.event_type == 'order_created':
self._handle_order_created(event)
elif event.event_type == 'order_updated':
self._handle_order_updated(event)
elif event.event_type == 'order_cancelled':
self._handle_order_cancelled(event)
def _handle_payment_event(self, event_data: Dict[str, Any]):
"""处理支付事件"""
event = PaymentEvent(**event_data)
if event.event_type == 'payment_completed':
self._handle_payment_completed(event)
elif event.event_type == 'payment_failed':
self._handle_payment_failed(event)
def _handle_order_created(self, event: OrderEvent):
"""处理订单创建"""
logger.info(f"处理订单创建: {event.order_id}")
# 1. 扣减库存
self._deduct_inventory(event.order_id, event.items)
# 2. 计算积分
self._calculate_points(event.user_id, event.total_amount)
# 3. 更新订单状态
self._update_order_status(event.order_id, 'confirmed')
logger.info(f"订单创建处理完成: {event.order_id}")
def _handle_order_updated(self, event: OrderEvent):
"""处理订单更新"""
logger.info(f"处理订单更新: {event.order_id}")
# 订单更新逻辑
time.sleep(0.1)
def _handle_order_cancelled(self, event: OrderEvent):
"""处理订单取消"""
logger.info(f"处理订单取消: {event.order_id}")
# 1. 恢复库存
self._restore_inventory(event.order_id, event.items)
# 2. 更新订单状态
self._update_order_status(event.order_id, 'cancelled')
logger.info(f"订单取消处理完成: {event.order_id}")
def _handle_payment_completed(self, event: PaymentEvent):
"""处理支付完成"""
logger.info(f"处理支付完成: {event.payment_id}, 订单: {event.order_id}")
# 1. 更新订单支付状态
self._update_order_payment_status(event.order_id, 'paid')
# 2. 触发订单处理
self._handle_order_created_after_payment(event.order_id)
logger.info(f"支付完成处理完成: {event.payment_id}")
def _handle_payment_failed(self, event: PaymentEvent):
"""处理支付失败"""
logger.info(f"处理支付失败: {event.payment_id}, 订单: {event.order_id}")
# 更新订单支付状态
self._update_order_payment_status(event.order_id, 'payment_failed')
logger.info(f"支付失败处理完成: {event.payment_id}")
def _deduct_inventory(self, order_id: int, items: list):
"""扣减库存"""
logger.info(f"为订单 {order_id} 扣减库存")
for item in items:
logger.info(f"扣减商品 {item.get('product_id')} 库存 {item.get('quantity')}")
time.sleep(0.05)
def _restore_inventory(self, order_id: int, items: list):
"""恢复库存"""
logger.info(f"为订单 {order_id} 恢复库存")
for item in items:
logger.info(f"恢复商品 {item.get('product_id')} 库存 {item.get('quantity')}")
time.sleep(0.05)
def _calculate_points(self, user_id: int, amount: float):
"""计算积分"""
points = int(amount)
logger.info(f"为用户 {user_id} 计算积分 {points}")
time.sleep(0.1)
def _update_order_status(self, order_id: int, status: str):
"""更新订单状态"""
logger.info(f"更新订单 {order_id} 状态为 {status}")
time.sleep(0.05)
def _update_order_payment_status(self, order_id: int, status: str):
"""更新订单支付状态"""
logger.info(f"更新订单 {order_id} 支付状态为 {status}")
time.sleep(0.05)
def _handle_order_created_after_payment(self, order_id: int):
"""支付后处理订单创建"""
logger.info(f"支付后处理订单创建: {order_id}")
# 这里可以执行支付后的特定逻辑
def _send_to_dlq(self, msg, error: str):
"""发送到死信队列"""
try:
# 在实际项目中，这里应该将失败消息发送到专门的死信队列
logger.error(f"消息发送到死信队列: {msg.topic()}[{msg.partition()}]@{msg.offset()}, 错误: {error}")
except Exception as e:
logger.error(f"发送到死信队列失败: {e}")
def stop_consuming(self):
"""停止消费"""
self.running = False
self.consumer.close()
logger.info("订单事件消费者已停止")
if __name__ == '__main__':
consumer = OrderEventConsumer()
try:
consumer.start_consuming()
except KeyboardInterrupt:
pass
python
# consumer/analytics_consumer.py
import json
import logging
import time
from datetime import datetime
from typing import Dict, Any
from confluent_kafka import Consumer, KafkaError
from schemas.events import OrderEvent, PaymentEvent, UserEvent, ProductEvent
logger = logging.getLogger(__name__)
class AnalyticsConsumer:
"""分析事件消费者"""
def __init__(self, bootstrap_servers: str = 'localhost:9092', group_id: str = 'analytics-processor'):
self.bootstrap_servers = bootstrap_servers
self.group_id = group_id
self.consumer = self._create_consumer()
self.running = False
# 订阅所有事件主题
self.topics = ['order_events', 'payment_events', 'user_events', 'product_events']
def _create_consumer(self) -> Consumer:
"""创建Kafka消费者"""
config = {
'bootstrap.servers': self.bootstrap_servers,
'group.id': self.group_id,
'auto.offset.reset': 'earliest',
'enable.auto.commit': False,
'max.poll.interval.ms': 300000,
}
consumer = Consumer(config)
consumer.subscribe(self.topics)
return consumer
def start_consuming(self):
"""开始消费事件"""
self.running = True
logger.info("分析事件消费者开始监听...")
try:
while self.running:
msg = self.consumer.poll(1.0)
if msg is None:
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
continue
else:
logger.error(f"消费者错误: {msg.error()}")
continue
try:
self._process_message(msg)
self.consumer.commit(asynchronous=False)
except Exception as e:
logger.error(f"分析事件处理失败: {e}")
except KeyboardInterrupt:
logger.info("收到中断信号，停止消费...")
except Exception as e:
logger.error(f"分析消费者异常: {e}")
finally:
self.stop_consuming()
def _process_message(self, msg):
"""处理分析事件"""
try:
event_data = json.loads(msg.value().decode('utf-8'))
event_type = event_data.get('event_type')
topic = msg.topic()
logger.info(f"处理分析事件: {event_type} from {topic}")
if topic == 'order_events':
self._analyze_order_event(event_data)
elif topic == 'payment_events':
self._analyze_payment_event(event_data)
elif topic == 'user_events':
self._analyze_user_event(event_data)
elif topic == 'product_events':
self._analyze_product_event(event_data)
# 批量更新分析数据（在实际项目中）
self._batch_update_analytics()
except Exception as e:
logger.error(f"分析事件处理异常: {e}")
raise
def _analyze_order_event(self, event_data: Dict[str, Any]):
"""分析订单事件"""
event = OrderEvent(**event_data)
if event.event_type == 'order_created':
# 记录订单创建指标
self._record_order_metrics(event)
# 用户行为分析
self._analyze_user_behavior(event.user_id, 'order_created', {
'order_id': event.order_id,
'amount': event.total_amount
})
elif event.event_type == 'order_cancelled':
# 记录订单取消指标
self._record_cancellation_metrics(event)
def _analyze_payment_event(self, event_data: Dict[str, Any]):
"""分析支付事件"""
event = PaymentEvent(**event_data)
if event.event_type == 'payment_completed':
# 记录支付成功指标
self._record_payment_success_metrics(event)
# 收入统计
self._record_revenue_metrics(event)
elif event.event_type == 'payment_failed':
# 记录支付失败指标
self._record_payment_failure_metrics(event)
def _analyze_user_event(self, event_data: Dict[str, Any]):
"""分析用户事件"""
event = UserEvent(**event_data)
if event.event_type == 'user_registered':
# 新用户注册统计
self._record_user_registration(event)
elif event.event_type == 'user_logged_in':
# 用户登录统计
self._record_user_login(event)
def _analyze_product_event(self, event_data: Dict[str, Any]):
"""分析商品事件"""
event = ProductEvent(**event_data)
if event.event_type == 'product_viewed':
# 商品浏览统计
self._record_product_view(event)
def _record_order_metrics(self, event: OrderEvent):
"""记录订单指标"""
logger.info(f"记录订单指标: 订单 {event.order_id}, 金额 {event.total_amount}")
# 实际应该写入时序数据库或分析数据库
time.sleep(0.02)
def _record_cancellation_metrics(self, event: OrderEvent):
"""记录取消指标"""
logger.info(f"记录取消指标: 订单 {event.order_id}")
time.sleep(0.02)
def _record_payment_success_metrics(self, event: PaymentEvent):
"""记录支付成功指标"""
logger.info(f"记录支付成功指标: 支付 {event.payment_id}")
time.sleep(0.02)
def _record_payment_failure_metrics(self, event: PaymentEvent):
"""记录支付失败指标"""
logger.info(f"记录支付失败指标: 支付 {event.payment_id}")
time.sleep(0.02)
def _record_revenue_metrics(self, event: PaymentEvent):
"""记录收入指标"""
logger.info(f"记录收入指标: 金额 {event.amount}")
time.sleep(0.01)
def _record_user_registration(self, event: UserEvent):
"""记录用户注册"""
logger.info(f"记录用户注册: 用户 {event.user_id}")
time.sleep(0.01)
def _record_user_login(self, event: UserEvent):
"""记录用户登录"""
logger.info(f"记录用户登录: 用户 {event.user_id}")
time.sleep(0.01)
def _record_product_view(self, event: ProductEvent):
"""记录商品浏览"""
logger.info(f"记录商品浏览: 商品 {event.product_id}")
time.sleep(0.01)
def _analyze_user_behavior(self, user_id: int, action: str, data: Dict[str, Any]):
"""分析用户行为"""
logger.info(f"分析用户行为: 用户 {user_id}, 动作 {action}")
time.sleep(0.05)
def _batch_update_analytics(self):
"""批量更新分析数据"""
# 在实际项目中，这里会批量更新分析数据库
# 例如：每100条消息或每10秒更新一次
pass
def stop_consuming(self):
"""停止消费"""
self.running = False
self.consumer.close()
logger.info("分析事件消费者已停止")
if __name__ == '__main__':
consumer = AnalyticsConsumer()
try:
consumer.start_consuming()
except KeyboardInterrupt:
pass
python
# consumer/analytics_consumer.py
import json
import logging
import time
from datetime import datetime
from typing import Dict, Any
from confluent_kafka import Consumer, KafkaError
from schemas.events import OrderEvent, PaymentEvent, UserEvent, ProductEvent
logger = logging.getLogger(__name__)
class AnalyticsConsumer:
"""分析事件消费者"""
def __init__(self, bootstrap_servers: str = 'localhost:9092', group_id: str = 'analytics-processor'):
self.bootstrap_servers = bootstrap_servers
self.group_id = group_id
self.consumer = self._create_consumer()
self.running = False
# 订阅所有事件主题
self.topics = ['order_events', 'payment_events', 'user_events', 'product_events']
def _create_consumer(self) -> Consumer:
"""创建Kafka消费者"""
config = {
'bootstrap.servers': self.bootstrap_servers,
'group.id': self.group_id,
'auto.offset.reset': 'earliest',
'enable.auto.commit': False,
'max.poll.interval.ms': 300000,
}
consumer = Consumer(config)
consumer.subscribe(self.topics)
return consumer
def start_consuming(self):
"""开始消费事件"""
self.running = True
logger.info("分析事件消费者开始监听...")
try:
while self.running:
msg = self.consumer.poll(1.0)
if msg is None:
continue
if msg.error():
if msg.error().code() == KafkaError._PARTITION_EOF:
continue
else:
logger.error(f"消费者错误: {msg.error()}")
continue
try:
self._process_message(msg)
self.consumer.commit(asynchronous=False)
except Exception as e:
logger.error(f"分析事件处理失败: {e}")
except KeyboardInterrupt:
logger.info("收到中断信号，停止消费...")
except Exception as e:
logger.error(f"分析消费者异常: {e}")
finally:
self.stop_consuming()
def _process_message(self, msg):
"""处理分析事件"""
try:
event_data = json.loads(msg.value().decode('utf-8'))
event_type = event_data.get('event_type')
topic = msg.topic()
logger.info(f"处理分析事件: {event_type} from {topic}")
if topic == 'order_events':
self._analyze_order_event(event_data)
elif topic == 'payment_events':
self._analyze_payment_event(event_data)
elif topic == 'user_events':
self._analyze_user_event(event_data)
elif topic == 'product_events':
self._analyze_product_event(event_data)
# 批量更新分析数据（在实际项目中）
self._batch_update_analytics()
except Exception as e:
logger.error(f"分析事件处理异常: {e}")
raise
def _analyze_order_event(self, event_data: Dict[str, Any]):
"""分析订单事件"""
event = OrderEvent(**event_data)
if event.event_type == 'order_created':
# 记录订单创建指标
self._record_order_metrics(event)
# 用户行为分析
self._analyze_user_behavior(event.user_id, 'order_created', {
'order_id': event.order_id,
'amount': event.total_amount
})
elif event.event_type == 'order_cancelled':
# 记录订单取消指标
self._record_cancellation_metrics(event)
def _analyze_payment_event(self, event_data: Dict[str, Any]):
"""分析支付事件"""
event = PaymentEvent(**event_data)
if event.event_type == 'payment_completed':
# 记录支付成功指标
self._record_payment_success_metrics(event)
# 收入统计
self._record_revenue_metrics(event)
elif event.event_type == 'payment_failed':
# 记录支付失败指标
self._record_payment_failure_metrics(event)
def _analyze_user_event(self, event_data: Dict[str, Any]):
"""分析用户事件"""
event = UserEvent(**event_data)
if event.event_type == 'user_registered':
# 新用户注册统计
self._record_user_registration(event)
elif event.event_type == 'user_logged_in':
# 用户登录统计
self._record_user_login(event)
def _analyze_product_event(self, event_data: Dict[str, Any]):
"""分析商品事件"""
event = ProductEvent(**event_data)
if event.event_type == 'product_viewed':
# 商品浏览统计
self._record_product_view(event)
def _record_order_metrics(self, event: OrderEvent):
"""记录订单指标"""
logger.info(f"记录订单指标: 订单 {event.order_id}, 金额 {event.total_amount}")
# 实际应该写入时序数据库或分析数据库
time.sleep(0.02)
def _record_cancellation_metrics(self, event: OrderEvent):
"""记录取消指标"""
logger.info(f"记录取消指标: 订单 {event.order_id}")
time.sleep(0.02)
def _record_payment_success_metrics(self, event: PaymentEvent):
"""记录支付成功指标"""
logger.info(f"记录支付成功指标: 支付 {event.payment_id}")
time.sleep(0.02)
def _record_payment_failure_metrics(self, event: PaymentEvent):
"""记录支付失败指标"""
logger.info(f"记录支付失败指标: 支付 {event.payment_id}")
time.sleep(0.02)
def _record_revenue_metrics(self, event: PaymentEvent):
"""记录收入指标"""
logger.info(f"记录收入指标: 金额 {event.amount}")
time.sleep(0.01)
def _record_user_registration(self, event: UserEvent):
"""记录用户注册"""
logger.info(f"记录用户注册: 用户 {event.user_id}")
time.sleep(0.01)
def _record_user_login(self, event: UserEvent):
"""记录用户登录"""
logger.info(f"记录用户登录: 用户 {event.user_id}")
time.sleep(0.01)
def _record_product_view(self, event: ProductEvent):
"""记录商品浏览"""
logger.info(f"记录商品浏览: 商品 {event.product_id}")
time.sleep(0.01)
def _analyze_user_behavior(self, user_id: int, action: str, data: Dict[str, Any]):
"""分析用户行为"""
logger.info(f"分析用户行为: 用户 {user_id}, 动作 {action}")
time.sleep(0.05)
def _batch_update_analytics(self):
"""批量更新分析数据"""
# 在实际项目中，这里会批量更新分析数据库
# 例如：每100条消息或每10秒更新一次
pass
def stop_consuming(self):
"""停止消费"""
self.running = False
self.consumer.close()
logger.info("分析事件消费者已停止")
if __name__ == '__main__':
consumer = AnalyticsConsumer()
try:
consumer.start_consuming()
except KeyboardInterrupt:
pass
python
# 使用示例 - API接口
from flask import Flask, request, jsonify
from producer.event_producer import EventProducer
from producer.kafka_producer import KafkaEventProducer
import logging
import uuid
app = Flask(__name__)
# 创建事件生产者
kafka_producer = KafkaEventProducer()
event_producer = EventProducer(kafka_producer)
@app.route('/api/orders', methods=['POST'])
def create_order():
"""创建订单接口"""
try:
data = request.get_json()
# 1. 创建订单（同步）
order_id = create_order_in_db(data)
order_data = {
'id': order_id,
'user_id': data['user_id'],
'total_amount': data['total_amount'],
'items': data.get('items', []),
'metadata': data.get('metadata', {})
}
# 2. 发送订单创建事件
event_producer.send_order_created(order_data)
return jsonify({
'success': True,
'order_id': order_id,
'message': '订单创建成功'
})
except Exception as e:
logging.error(f"订单创建失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
@app.route('/api/payments/callback', methods=['POST'])
def payment_callback():
"""支付回调接口"""
try:
data = request.get_json()
payment_data = {
'id': data['payment_id'],
'order_id': data['order_id'],
'amount': data['amount'],
'method': data.get('method', 'alipay'),
'gateway_response': data.get('gateway_response'),
'metadata': data.get('metadata', {})
}
if data['status'] == 'success':
event_producer.send_payment_completed(payment_data)
else:
payment_data.update({
'error_code': data.get('error_code'),
'error_message': data.get('error_message')
})
event_producer.send_payment_failed(payment_data)
return jsonify({'success': True})
except Exception as e:
logging.error(f"支付回调处理失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
@app.route('/api/users/register', methods=['POST'])
def register_user():
"""用户注册接口"""
try:
data = request.get_json()
user_id = create_user_in_db(data)
user_data = {
'id': user_id,
'metadata': data.get('metadata', {})
}
# 发送用户注册事件
event_producer.send_user_registered(user_data)
return jsonify({
'success': True,
'user_id': user_id,
'message': '用户注册成功'
})
except Exception as e:
logging.error(f"用户注册失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
@app.route('/api/products/<int:product_id>/view', methods=['POST'])
def track_product_view(product_id):
"""追踪商品浏览"""
try:
data = request.get_json()
product_data = {
'id': product_id,
'user_id': data.get('user_id'),
'category_id': data.get('category_id'),
'price': data.get('price'),
'metadata': data.get('metadata', {})
}
# 发送商品浏览事件
event_producer.send_product_viewed(product_data)
return jsonify({'success': True})
except Exception as e:
logging.error(f"商品浏览追踪失败: {e}")
return jsonify({
'success': False,
'error': str(e)
}), 500
def create_order_in_db(data):
"""模拟创建订单到数据库"""
return 12345 # 返回订单ID
def create_user_in_db(data):
"""模拟创建用户到数据库"""
return 1001 # 返回用户ID
@app.teardown_appcontext
def close_producer(exception=None):
"""关闭生产者"""
kafka_producer.close()
if __name__ == '__main__':
app.run(debug=True)特性 Celery + Redis RabbitMQ Apache Kafka
消息模型 任务队列 消息代理 事件流平台
持久化 可选 支持 强持久化
消息顺序 不保证 队列内保证 分区内保证
吞吐量 中等 中等 非常高
延迟 低 低 低到中等
适用场景 异步任务、定时任务 应用解耦、复杂路由 事件流、数据管道
复杂度 低 中等 高
数据重放 不支持 有限支持 完整支持
生态系统 Python生态 多语言支持 完整流生态