RabbitMQ消息队列实战项目
项目一：电商订单处理系统
项目架构
text
[Web应用] → [订单队列] → [订单处理服务] → [库存队列] → [库存服务]
↓ ↓
[死信队列] [通知队列] → [短信/邮件服务]
核心代码实现
1. 环境配置
python
# config.py
import pika
import json
from typing import Dict, Any
class RabbitMQConfig:
# 连接配置
RABBITMQ_HOST = 'localhost'
RABBITMQ_PORT = 5672
RABBITMQ_USER = 'guest'
RABBITMQ_PASSWORD = 'guest'
VIRTUAL_HOST = '/'
# 交换机配置
ORDER_EXCHANGE = 'order.exchange'
INVENTORY_EXCHANGE = 'inventory.exchange'
NOTIFICATION_EXCHANGE = 'notification.exchange'
# 队列配置
ORDER_QUEUE = 'order.queue'
ORDER_DLX_QUEUE = 'order.dlx.queue'
INVENTORY_QUEUE = 'inventory.queue'
EMAIL_QUEUE = 'email.queue'
SMS_QUEUE = 'sms.queue'
# 路由键
ORDER_ROUTING_KEY = 'order.create'
INVENTORY_ROUTING_KEY = 'inventory.check'
EMAIL_ROUTING_KEY = 'notification.email'
SMS_ROUTING_KEY = 'notification.sms'
2. 连接管理器
python
# connection_manager.py
import pika
from typing import Optional
import threading
import logging
class RabbitMQConnectionManager:
"""RabbitMQ连接管理器（单例模式）"""
_instance = None
_lock = threading.Lock()
def __new__(cls):
with cls._lock:
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._initialized = False
return cls._instance
def __init__(self):
if self._initialized:
return
self._initialized = True
self.connection: Optional[pika.BlockingConnection] = None
self.channel: Optional[pika.adapters.blocking_connection.BlockingChannel] = None
self.logger = logging.getLogger(__name__)
def connect(self):
"""建立连接"""
try:
credentials = pika.PlainCredentials(
RabbitMQConfig.RABBITMQ_USER,
RabbitMQConfig.RABBITMQ_PASSWORD
)
parameters = pika.ConnectionParameters(
host=RabbitMQConfig.RABBITMQ_HOST,
port=RabbitMQConfig.RABBITMQ_PORT,
virtual_host=RabbitMQConfig.VIRTUAL_HOST,
credentials=credentials,
heartbeat=600,
blocked_connection_timeout=300
)
self.connection = pika.BlockingConnection(parameters)
self.channel = self.connection.channel()
self.logger.info("RabbitMQ连接成功")
except Exception as e:
self.logger.error(f"RabbitMQ连接失败: {e}")
raise
def get_channel(self):
"""获取channel"""
if not self.connection or self.connection.is_closed:
self.connect()
return self.channel
def close(self):
"""关闭连接"""
if self.connection and not self.connection.is_closed:
self.connection.close()
self.logger.info("RabbitMQ连接已关闭")
3. 消息发布者
python
# publisher.py
import json
import uuid
from datetime import datetime
from typing import Dict, Any
import pika
class OrderPublisher:
"""订单消息发布者"""
def __init__(self, connection_manager):
self.connection_manager = connection_manager
self.channel = connection_manager.get_channel()
self.setup_exchanges()
def setup_exchanges(self):
"""设置交换机"""
# 订单交换机
self.channel.exchange_declare(
exchange=RabbitMQConfig.ORDER_EXCHANGE,
exchange_type='direct',
durable=True
)
# 死信交换机
self.channel.exchange_declare(
exchange='order.dlx.exchange',
exchange_type='direct',
durable=True
)
# 队列声明
arguments = {
'x-dead-letter-exchange': 'order.dlx.exchange',
'x-dead-letter-routing-key': 'order.dlx',
'x-message-ttl': 30000 # 30秒过期
}
self.channel.queue_declare(
queue=RabbitMQConfig.ORDER_QUEUE,
durable=True,
arguments=arguments
)
# 死信队列
self.channel.queue_declare(
queue=RabbitMQConfig.ORDER_DLX_QUEUE,
durable=True
)
# 绑定
self.channel.queue_bind(
exchange=RabbitMQConfig.ORDER_EXCHANGE,
queue=RabbitMQConfig.ORDER_QUEUE,
routing_key=RabbitMQConfig.ORDER_ROUTING_KEY
)
self.channel.queue_bind(
exchange='order.dlx.exchange',
queue=RabbitMQConfig.ORDER_DLX_QUEUE,
routing_key='order.dlx'
)
def publish_order(self, order_data: Dict[str, Any]):
"""发布订单消息"""
message_id = str(uuid.uuid4())
message = {
'message_id': message_id,
'timestamp': datetime.now().isoformat(),
'data': order_data
}
# 消息属性
properties = pika.BasicProperties(
delivery_mode=2, # 持久化
message_id=message_id,
content_type='application/json',
timestamp=int(datetime.now().timestamp())
)
try:
self.channel.basic_publish(
exchange=RabbitMQConfig.ORDER_EXCHANGE,
routing_key=RabbitMQConfig.ORDER_ROUTING_KEY,
body=json.dumps(message),
properties=properties
)
print(f"订单消息已发布: {message_id}")
return message_id
except Exception as e:
print(f"消息发布失败: {e}")
raise
4. 消息消费者
python
# consumer.py
import json
import time
from typing import Callable
import pika
class OrderConsumer:
"""订单消息消费者"""
def __init__(self, connection_manager):
self.connection_manager = connection_manager
self.channel = connection_manager.get_channel()
self.setup_qos()
def setup_qos(self):
"""设置QoS"""
# 每次只处理一条消息
self.channel.basic_qos(prefetch_count=1)
def process_order(self, ch, method, properties, body):
"""处理订单消息"""
try:
message = json.loads(body)
order_data = message['data']
print(f"处理订单: {order_data['order_id']}")
# 模拟订单处理
time.sleep(2)
# 处理成功，确认消息
ch.basic_ack(delivery_tag=method.delivery_tag)
print(f"订单处理完成: {order_data['order_id']}")
except Exception as e:
print(f"订单处理失败: {e}")
# 判断是否重新入队
if method.redelivered:
# 已经重试过，拒绝消息
ch.basic_reject(delivery_tag=method.delivery_tag, requeue=False)
else:
# 第一次失败，重新入队
ch.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
def start_consuming(self):
"""开始消费"""
self.channel.basic_consume(
queue=RabbitMQConfig.ORDER_QUEUE,
on_message_callback=self.process_order,
auto_ack=False
)
print("开始监听订单队列...")
self.channel.start_consuming()
5. 异步消息处理
python
# async_processor.py
import asyncio
import aio_pika
from typing import Dict, Any
import json
class AsyncOrderProcessor:
"""异步订单处理器"""
def __init__(self, rabbitmq_url: str):
self.rabbitmq_url = rabbitmq_url
self.connection = None
self.channel = None
async def connect(self):
"""建立异步连接"""
self.connection = await aio_pika.connect_robust(
self.rabbitmq_url,
reconnect_interval=5
)
self.channel = await self.connection.channel()
# 设置QoS
await self.channel.set_qos(prefetch_count=10)
async def process_order(self, message: aio_pika.IncomingMessage):
"""处理订单"""
async with message.process():
try:
order_data = json.loads(message.body)
print(f"异步处理订单: {order_data['order_id']}")
# 模拟异步处理
await asyncio.sleep(1)
# 调用其他服务
await self.check_inventory(order_data)
await self.send_notification(order_data)
print(f"订单处理完成: {order_data['order_id']}")
except Exception as e:
print(f"异步处理失败: {e}")
# 消息会被自动拒绝并重新入队
raise
async def check_inventory(self, order_data: Dict[str, Any]):
"""检查库存"""
# 发送库存检查消息
inventory_exchange = await self.channel.declare_exchange(
RabbitMQConfig.INVENTORY_EXCHANGE,
aio_pika.ExchangeType.DIRECT
)
await inventory_exchange.publish(
aio_pika.Message(
body=json.dumps(order_data).encode(),
delivery_mode=aio_pika.DeliveryMode.PERSISTENT
),
routing_key=RabbitMQConfig.INVENTORY_ROUTING_KEY
)
async def send_notification(self, order_data: Dict[str, Any]):
"""发送通知"""
notification_exchange = await self.channel.declare_exchange(
RabbitMQConfig.NOTIFICATION_EXCHANGE,
aio_pika.ExchangeType.FANOUT
)
await notification_exchange.publish(
aio_pika.Message(
body=json.dumps({
'type': 'order_confirmation',
'data': order_data
}).encode()
),
routing_key=''
)
async def start(self):
"""启动消费者"""
await self.connect()
queue = await self.channel.declare_queue(
RabbitMQConfig.ORDER_QUEUE,
durable=True
)
await queue.consume(self.process_order)
print("异步订单处理器已启动...")
项目二：实时日志监控系统
系统架构
text
[应用服务] → [日志队列] → [日志处理器] → [Elasticsearch]
↓
[告警队列] → [告警服务] → [钉钉/邮件]
日志收集器实现
python
# log_collector.py
import logging
import json
import socket
from datetime import datetime
import pika
class RabbitMQLogHandler(logging.Handler):
"""RabbitMQ日志处理器"""
def __init__(self, connection_manager, exchange_name='logs.exchange'):
super().__init__()
self.connection_manager = connection_manager
self.channel = connection_manager.get_channel()
self.exchange_name = exchange_name
self.hostname = socket.gethostname()
# 声明交换机
self.channel.exchange_declare(
exchange=exchange_name,
exchange_type='topic',
durable=True
)
def emit(self, record):
"""发送日志"""
try:
log_entry = {
'timestamp': datetime.utcnow().isoformat(),
'hostname': self.hostname,
'level': record.levelname,
'logger': record.name,
'message': self.format(record),
'module': record.module,
'line': record.lineno
}
# 根据日志级别设置路由键
routing_key = f"log.{record.levelname.lower()}"
self.channel.basic_publish(
exchange=self.exchange_name,
routing_key=routing_key,
body=json.dumps(log_entry),
properties=pika.BasicProperties(
delivery_mode=1, # 非持久化
content_type='application/json'
)
)
except Exception:
self.handleError(record)
class LogProcessor:
"""日志处理器"""
def __init__(self, connection_manager):
self.connection_manager = connection_manager
self.channel = connection_manager.get_channel()
self.error_count = 0
self.setup_queues()
def setup_queues(self):
"""设置队列"""
self.channel.exchange_declare(
exchange='logs.exchange',
exchange_type='topic',
durable=True
)
# 错误日志队列
self.channel.queue_declare(queue='logs.error', durable=True)
self.channel.queue_bind(
exchange='logs.exchange',
queue='logs.error',
routing_key='log.error'
)
# 警告日志队列
self.channel.queue_declare(queue='logs.warning', durable=True)
self.channel.queue_bind(
exchange='logs.exchange',
queue='logs.warning',
routing_key='log.warning'
)
# 所有日志队列（用于存储）
self.channel.queue_declare(queue='logs.all', durable=True)
self.channel.queue_bind(
exchange='logs.exchange',
queue='logs.all',
routing_key='log.*'
)
def process_error_log(self, ch, method, properties, body):
"""处理错误日志"""
log_data = json.loads(body)
self.error_count += 1
print(f"错误日志: {log_data['message']}")
# 触发告警
if self.error_count >= 5: # 5个错误触发告警
self.send_alert(log_data)
self.error_count = 0
ch.basic_ack(delivery_tag=method.delivery_tag)
def send_alert(self, error_log):
"""发送告警"""
alert_data = {
'type': 'error_alert',
'message': f"连续错误超过阈值: {error_log['message']}",
'timestamp': datetime.now().isoformat()
}
self.channel.basic_publish(
exchange='alert.exchange',
routing_key='alert.high',
body=json.dumps(alert_data)
)
项目三：任务队列系统（Celery集成）
Celery配置
python
# celery_app.py
from celery import Celery
from kombu import Exchange, Queue
app = Celery(
'tasks',
broker='amqp://guest:guest@localhost:5672//',
backend='redis://localhost:6379/0'
)
# 配置队列
app.conf.task_queues = (
Queue('high_priority', Exchange('tasks', type='direct'), routing_key='high'),
Queue('default', Exchange('tasks', type='direct'), routing_key='default'),
Queue('low_priority', Exchange('tasks', type='direct'), routing_key='low'),
)
app.conf.task_routes = {
'tasks.process_image': {'queue': 'high_priority'},
'tasks.send_email': {'queue': 'default'},
'tasks.generate_report': {'queue': 'low_priority'},
}
app.conf.task_default_queue = 'default'
app.conf.task_default_exchange = 'tasks'
app.conf.task_default_routing_key = 'default'
# 任务实现
@app.task(bind=True, max_retries=3, default_retry_delay=60)
def process_image(self, image_path):
"""图像处理任务"""
try:
# 图像处理逻辑
result = image_processing(image_path)
return result
except Exception as exc:
raise self.retry(exc=exc)
@app.task
def send_email(to_address, subject, body):
"""发送邮件任务"""
# 邮件发送逻辑
email_service.send(to_address, subject, body)
@app.task
def generate_report(report_type, params):
"""生成报告任务"""
# 报告生成逻辑
return report_service.generate(report_type, params)
项目四：消息广播系统
实现发布/订阅模式
python
# pubsub_system.py
import pika
import json
from typing import List
class MessageBroadcaster:
"""消息广播系统"""
def __init__(self, connection_manager):
self.connection_manager = connection_manager
self.channel = connection_manager.get_channel()
def setup_fanout_exchange(self, exchange_name):
"""设置广播交换机"""
self.channel.exchange_declare(
exchange=exchange_name,
exchange_type='fanout',
durable=True
)
def publish_broadcast(self, exchange_name, message):
"""发布广播消息"""
self.channel.basic_publish(
exchange=exchange_name,
routing_key='', # fanout忽略routing key
body=json.dumps(message),
properties=pika.BasicProperties(
delivery_mode=2,
content_type='application/json'
)
)
def subscribe(self, exchange_name, queue_name='', callback=None):
"""订阅广播消息"""
# 创建临时队列
if not queue_name:
result = self.channel.queue_declare(queue='', exclusive=True)
queue_name = result.method.queue
else:
self.channel.queue_declare(queue=queue_name, durable=True)
# 绑定到广播交换机
self.channel.queue_bind(
exchange=exchange_name,
queue=queue_name
)
if callback:
self.channel.basic_consume(
queue=queue_name,
on_message_callback=callback,
auto_ack=True
)
# WebSocket集成示例
from fastapi import FastAPI, WebSocket
from fastapi.responses import HTMLResponse
app = FastAPI()
class WebSocketBroadcaster:
def __init__(self):
self.connections: List[WebSocket] = []
self.broadcaster = MessageBroadcaster(RabbitMQConnectionManager())
async def connect(self, websocket: WebSocket):
await websocket.accept()
self.connections.append(websocket)
def disconnect(self, websocket: WebSocket):
self.connections.remove(websocket)
async def broadcast_to_websockets(self, message):
for connection in self.connections:
try:
await connection.send_text(message)
except:
await self.disconnect(connection)
def on_rabbitmq_message(self, ch, method, properties, body):
"""处理RabbitMQ消息并推送到WebSocket"""
message = body.decode()
asyncio.create_task(self.broadcast_to_websockets(message))
监控和管理
健康检查与监控
python
# monitoring.py
import psutil
import time
from typing import Dict, Any
import pika
class RabbitMQMonitor:
"""RabbitMQ监控器"""
def __init__(self, connection_manager):
self.connection_manager = connection_manager
self.channel = connection_manager.get_channel()
self.metrics = {}
def get_queue_metrics(self, queue_name: str) -> Dict[str, Any]:
"""获取队列指标"""
queue = self.channel.queue_declare(
queue=queue_name,
passive=True # 不创建队列
)
return {
'queue_name': queue_name,
'message_count': queue.method.message_count,
'consumer_count': queue.method.consumer_count
}
def check_health(self) -> Dict[str, Any]:
"""健康检查"""
try:
# 检查连接
if not self.connection_manager.connection.is_open:
return {'status': 'unhealthy', 'reason': 'connection_closed'}
# 检查通道
if not self.channel.is_open:
return {'status': 'unhealthy', 'reason': 'channel_closed'}
# 检查系统资源
cpu_usage = psutil.cpu_percent()
memory_usage = psutil.virtual_memory().percent
if cpu_usage > 90 or memory_usage > 90:
return {
'status': 'degraded',
'cpu_usage': cpu_usage,
'memory_usage': memory_usage
}
return {
'status': 'healthy',
'cpu_usage': cpu_usage,
'memory_usage': memory_usage,
'uptime': time.time() - self.start_time
}
except Exception as e:
return {'status': 'unhealthy', 'reason': str(e)}
def collect_metrics(self):
"""收集指标"""
queues = ['order.queue', 'inventory.queue', 'notification.queue']
for queue in queues:
try:
metrics = self.get_queue_metrics(queue)
self.metrics[queue] = metrics
except Exception as e:
self.metrics[queue] = {'error': str(e)}
return self.metrics
这些实战项目涵盖了RabbitMQ的核心使用场景，包括：
1. 订单处理系统的异步解耦
2. 日志收集和监控
3. 任务队列的优先级处理
4. 消息广播和实时推送
5. 系统监控和健康检查