队列数据管理模块
在消息队列系统中,队列作为消息存储与转发的核心单元,其元数据管理与持久化是保证服务可靠性的关键。本篇介绍消息队列中间件中队列数据的内存管理、持久化存储、线程安全处理完整实现,代码结构与交换机数据管理保持统一,具备工业级可用性。
一、队列结构体定义
队列结构体用于封装队列核心属性,包含名称、持久化、排他、自动删除及扩展参数,并提供参数序列化与反序列化接口,便于数据库存储与读取。
cpp
struct MsgQueue{
using ptr = std::shared_ptr<MsgQueue>;
std::string name;
bool durable;
bool exclusive;
bool auto_delete;
google::protobuf::Map<std::string,std::string> args;
MsgQueue() = default;
MsgQueue(const std::string& qname,
bool qdurable,
bool qexclusive,
bool qauto_delete,
const google::protobuf::Map<std::string, std::string>& qargs)
: name(qname),
durable(qdurable),
exclusive(qexclusive),
auto_delete(qauto_delete),
args(qargs) {}
// 字符串参数解析到Map
void SetArgs(const std::string &str_args){
std::vector<std::string> sub_args;
StrHelper::split(str_args,"&",sub_args);
for(auto &str : sub_args){
size_t pos = str.find("=");
if( pos == std::string::npos){
ELOG("交换机解析格式非法:%s",str.c_str());
continue;
}
std::string key = str.substr(0,pos);
std::string value = str.substr(pos+1);
args[key] = value;
}
}
// Map参数序列化为字符串
std::string getArgs() const {
std::string result;
for(auto &it:args){
result += it.first + "=" + it.second + "&";
}
return result;
}
};
二、队列数据持久化实现
基于SQLite实现队列元数据持久化,负责队列数据的增、删、查、表创建与数据恢复,是服务重启后数据不丢失的核心。
cpp
using QueueMap = std::unordered_map<std::string,MsgQueue::ptr>;
class MsgQueueMapper{
public:
MsgQueueMapper(const std::string &dbfile) : _sql_helper(dbfile){
std::string path = FileHelper::parentDirectory(dbfile);
FileHelper::createDirectory(path);
_sql_helper.open();
createTable();
}
// 创建队列数据表
void createTable(){
std::stringstream sql;
sql << "create table if not exists queue_table(";
sql << "name varchar(32) primary key,";
sql << "durable int,";
sql << "exclusive int,";
sql << "auto_delete int,";
sql << "args varchar(128));";
assert(_sql_helper.exec(sql.str(),nullptr,nullptr));
}
// 删除数据表
void removeTable(){
std::string sql = "drop table if exists queue_table";
_sql_helper.exec(sql,nullptr,nullptr);
}
// 插入队列数据
bool insert(MsgQueue::ptr &queue){
std::stringstream sql;
sql << "insert into queue_table(name,durable,exclusive,auto_delete,args) values(";
sql << "'" << queue->name <<"',";
sql << (int)queue->durable <<",";
sql << (int)queue->exclusive <<",";
sql << (int)queue->auto_delete <<",";
sql << "'" << queue->getArgs() <<"');";
return _sql_helper.exec(sql.str(),nullptr,nullptr);
}
// 删除指定队列数据
void remove(const std::string &name){
std::string sql = "delete from queue_table where name = '" + name + "'";
_sql_helper.exec(sql,nullptr,nullptr);
}
// 从数据库恢复所有队列数据
QueueMap recovery() {
QueueMap result;
std::string sql = "select name,durable,exclusive,auto_delete,args from queue_table;";
_sql_helper.exec(sql,selectCallback,&result);
return result;
}
private:
// 数据读取回调函数
static int selectCallback(void* arg,int numcol,char** row,char** fields){
QueueMap *result = (QueueMap *)arg;
if(!row[0] || !row[1] || !row[2] || !row[3]) return 0;
MsgQueue::ptr mqp = std::make_shared<MsgQueue>();
mqp->name = row[0];
mqp->durable = (bool)std::stoi(row[1]);
mqp->exclusive = (bool)std::stoi(row[2]);
mqp->auto_delete = (bool)std::stoi(row[3]);
if(row[4]) mqp->SetArgs(row[4]);
result->insert(std::make_pair(mqp->name,mqp));
return 0;
}
private:
SqliteHelper _sql_helper;
};
三、线程安全的队列管理类
对外提供统一的队列操作接口,内部维护内存队列映射,所有公共接口加锁保证多线程安全,区分持久化与非持久化队列逻辑。
cpp
class MsgQueueManager {
public:
using ptr = std::shared_ptr<MsgQueueManager>;
MsgQueueManager(const std::string &dbfile):_mapper(dbfile) {
std::unique_lock<std::mutex> lock(_mutex);
_msg_queues = _mapper.recovery();
}
// 声明队列:存在则返回true,不存在则创建
bool declareQueue(const std::string &qname,
bool qdurable,
bool qexclusive,
bool qauto_delete,
const google::protobuf::Map<std::string,std::string> &qargs){
std::unique_lock<std::mutex> lock(_mutex);
auto it = _msg_queues.find(qname);
if( it != _msg_queues.end()){
return true;
}
MsgQueue::ptr mqp = std::make_shared<MsgQueue>();
mqp->name = qname;
mqp->durable = qdurable;
mqp->exclusive = qexclusive;
mqp->auto_delete = qauto_delete;
mqp->args = qargs;
if(qdurable == true){
if(!_mapper.insert(mqp)){
return false;
}
}
_msg_queues.insert(std::make_pair(qname,mqp));
return true;
}
// 删除队列:持久化队列同步删除数据库数据
void deleteQueue(const std::string &name){
std::unique_lock<std::mutex> lock(_mutex);
auto it = _msg_queues.find(name);
if( it == _msg_queues.end()) return;
if( it->second->durable == true){
_mapper.remove(name);
}
_msg_queues.erase(it);
}
// 查询指定队列
MsgQueue::ptr selectQueue(const std::string &name){
std::unique_lock<std::mutex> lock(_mutex);
auto it = _msg_queues.find(name);
if( it == _msg_queues.end()) return MsgQueue::ptr();
return it->second;
}
// 获取所有队列
QueueMap allQueues() {
std::unique_lock<std::mutex> lock(_mutex);
return _msg_queues;
}
// 判断队列是否存在
bool exists(const std::string &name){
std::unique_lock<std::mutex> lock(_mutex);
auto it = _msg_queues.find(name);
if(it == _msg::end()) return false;
return true;
}
// 获取队列数量
size_t size(){
std::unique_lock<std::mutex> lock(_mutex);
return _msg_queues.size();
}
// 清空所有队列与数据表
void clear(){
std::unique_lock<std::mutex> lock(_mutex);
_mapper.removeTable();
_msg_queues.clear();
}
private:
std::mutex _mutex;
MsgQueueMapper _mapper;
QueueMap _msg_queues;
};
四、模块单元测试(GTest)
对队列管理模块做全功能串联测试,覆盖声明、删除、查询、存在判断、非持久化验证、清空等逻辑。
cpp
#include "../mqserver/mq_queue.hpp"
#include <gtest/gtest.h>
#include <vector>
Fy_mq::MsgQueueManager::ptr mqmp;
class QueueTest : public testing::Environment
{
public:
virtual void SetUp() override
{
static std::once_flag flag;
std::call_once(flag, [&]() {
mqmp = std::make_shared<Fy_mq::MsgQueueManager>("./data/meta.db");
});
}
virtual void TearDown() override
{
mqmp->clear();
}
};
// 1. 声明4个队列
TEST(queue_test, step1_declare_4_queues) {
google::protobuf::Map<std::string, std::string> args;
args["k1"] = "v1";
mqmp->declareQueue("queue1", true, false, false, args);
mqmp->declareQueue("queue2", true, false, false, args);
mqmp->declareQueue("queue3", true, false, false, args);
mqmp->declareQueue("queue4", true, false, false, args);
ASSERT_EQ(mqmp->size(), 4);
}
// 2. 重复声明队列
TEST(queue_test, step2_duplicate_declare) {
google::protobuf::Map<std::string, std::string> args;
bool ret = mqmp->declareQueue("queue1", true, false, false, args);
ASSERT_TRUE(ret);
ASSERT_EQ(mqmp->size(), 4);
}
// 3. 存在判断与查询
TEST(queue_test, step3_exists_and_select) {
ASSERT_TRUE(mqmp->exists("queue1"));
ASSERT_TRUE(mqmp->exists("queue2"));
ASSERT_FALSE(mqmp->exists("queue_not_exist"));
auto q = mqmp->selectQueue("queue3");
ASSERT_NE(q, nullptr);
ASSERT_EQ(q->name, "queue3");
}
// 4. 删除队列
TEST(queue_test, step4_delete_2_queues) {
mqmp->deleteQueue("queue1");
mqmp->deleteQueue("queue2");
ASSERT_EQ(mqmp->size(), 2);
ASSERT_FALSE(mqmp->exists("queue1"));
ASSERT_FALSE(mqmp->exists("queue2"));
}
// 5. 获取所有队列
TEST(queue_test, step5_all_queues) {
auto queues = mqmp->allQueues();
ASSERT_EQ(queues.size(), 2);
}
// 6. 非持久化队列不恢复
TEST(queue_test, step7_non_durable_no_recovery) {
mqmp->clear();
google::protobuf::Map<std::string, std::string> args;
mqmp->declareQueue("temp_queue", false, false, false, args);
mqmp.reset();
mqmp = std::make_shared<Fy_mq::MsgQueueManager>("./data/meta.db");
ASSERT_FALSE(mqmp->exists("temp_queue"));
}
// 7. 清空所有数据
TEST(queue_test, step8_clear_all) {
mqmp->clear();
ASSERT_EQ(mqmp->size(), 0);
}
int main(int argc,char *argv[]){
testing::InitGoogleTest(&argc,argv);
testing::AddGlobalTestEnvironment(new QueueTest);
return RUN_ALL_TESTS();
}
运行结果:
bash
zhangwho@VM-0-2-ubuntu:~/mq/mqtest$ ./mq_queuetest
[==========] Running 7 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 7 tests from queue_test
[ RUN ] queue_test.step1_declare_4_queues
[ OK ] queue_test.step1_declare_4_queues (23 ms)
[ RUN ] queue_test.step2_duplicate_declare
[ OK ] queue_test.step2_duplicate_declare (0 ms)
[ RUN ] queue_test.step3_exists_and_select
[ OK ] queue_test.step3_exists_and_select (0 ms)
[ RUN ] queue_test.step4_delete_2_queues
[ OK ] queue_test.step4_delete_2_queues (12 ms)
[ RUN ] queue_test.step5_all_queues
[ OK ] queue_test.step5_all_queues (0 ms)
[ RUN ] queue_test.step7_non_durable_no_recovery
[ OK ] queue_test.step7_non_durable_no_recovery (12 ms)
[ RUN ] queue_test.step8_clear_all
[ OK ] queue_test.step8_clear_all (4 ms)
[----------] 7 tests from queue_test (51 ms total)
[----------] Global test environment tear-down
[==========] 7 tests from 1 test suite ran. (58 ms total)
[ PASSED ] 7 tests.
五、功能说明
- 持久化区分:持久化队列写入SQLite,重启自动恢复;非持久化队列仅存内存,重启消失。
- 线程安全:所有对外接口使用互斥锁保护,支持多线程并发访问。
- 幂等声明:同一队列多次声明不会重复创建,符合消息队列语义。
- 数据恢复:服务启动时自动从数据库加载队列数据。
- 完整接口:支持声明、删除、查询、判断存在、获取全部、清空等操作。
绑定数据管理模块
绑定是交换机与队列之间的关联关系,通过绑定键实现消息的路由转发。为了保证服务重启之后绑定关系不丢失,需要对绑定数据进行持久化管理。本篇文章介绍绑定数据结构定义、数据持久化、线程安全管理的完整实现,代码结构与交换机、队列模块保持一致。
一、绑定结构体定义
用于描述交换机与队列之间的绑定关系,包含交换机名称、队列名称、绑定键三个核心属性。
cpp
struct Binding{
using ptr = std::shared_ptr<Binding>;
std::string exchange_name;
std::string msgqueue_name;
std::string binding_key;
Binding() = default;
Binding(const std::string &ename,const std::string &qname,const std::string &key)
:exchange_name(ename),msgqueue_name(qname),binding_key(key){}
};
using MsgQueueBindMap = std::unordered_map<std::string,Binding::ptr>;
using BindingMap = std::unordered_map<std::string,MsgQueueBindMap>;
二、绑定数据持久化实现
基于 SQLite 实现绑定数据的持久化存储,提供绑定新增、删除、按交换机删除、按队列删除、数据恢复等功能。
cpp
class BindingMapper{
public:
BindingMapper(const std::string &dbfile): _sql_helper(dbfile){
std::string path = FileHelper::parentDirectory(dbfile);
FileHelper::createDirectory(path);
_sql_helper.open();
createTable();
}
void createTable(){
std::stringstream sql;
sql << "create table if not exists binding_table(";
sql << "exchange_name varchar(32), ";
sql << "msgqueue_name varchar(32), ";
sql << "binding_key varchar(128),";
sql << "primary key(exchange_name, msgqueue_name));";
assert(_sql_helper.exec(sql.str(), nullptr, nullptr));
}
void removeTable(){
std::string sql = "drop table if exists binding_table;";
_sql_helper.exec(sql, nullptr, nullptr);
}
bool insert(Binding::ptr &binding){
std::stringstream sql;
sql << "insert into binding_table(exchange_name,msgqueue_name,binding_key) values(";
sql << "'" << binding->exchange_name << "',";
sql << "'" << binding->msgqueue_name << "',";
sql << "'" << binding->binding_key << "');";
return _sql_helper.exec(sql.str(),nullptr,nullptr);
}
void remove(const std::string &ename,const std::string &qname){
std::stringstream sql;
sql << "delete from binding_table where ";
sql << "exchange_name = '" << ename << "' and ";
sql << "msgqueue_name = '" << qname << "';";
_sql_helper.exec(sql.str(),nullptr,nullptr);
}
void removeExchangeBindings(const std::string &ename){
std::stringstream sql;
sql << "delete from binding_table where ";
sql << "exchange_name = '" << ename << "';";
_sql_helper.exec(sql.str(),nullptr,nullptr);
}
void removeMsgqueueBindings(const std::string &qname){
std::stringstream sql;
sql << "delete from binding_table where ";
sql << "msgqueue_name = '" << qname << "';";
_sql_helper.exec(sql.str(),nullptr,nullptr);
}
BindingMap recovery() {
BindingMap result;
std::string sql = "select exchange_name , msgqueue_name , binding_key from binding_table;";
_sql_helper.exec(sql,selectCallback,&result);
return result;
}
private:
static int selectCallback(void* arg,int numcol,char** row,char** fields){
if (!row[0] || !row[1] || !row[2]) return 0;
BindingMap *result = (BindingMap *)arg;
auto& qmap = (*result)[row[0]];
Binding::ptr bdp = std::make_shared<Binding>();
bdp->exchange_name = row[0];
bdp->msgqueue_name = row[1];
bdp->binding_key = row[2];
qmap.insert(std::make_pair(bdp->msgqueue_name,bdp));
return 0;
}
private:
SqliteHelper _sql_helper;
};
三、线程安全绑定管理类
对外提供统一的绑定操作接口,所有操作均加锁保证线程安全,区分持久化与非持久化绑定逻辑。
cpp
class BindingManager {
public:
using ptr = std::shared_ptr<BindingManager>;
BindingManager(const std::string &dbfile):_mapper(dbfile){
std::unique_lock<std::mutex> lock(_mutex);
_bindings = _mapper.recovery();
}
bool bind(const std::string &ename,const std::string &qname,const std::string &key,bool durable){
std::unique_lock<std::mutex> lock(_mutex);
auto it = _bindings.find(ename);
if(it != _bindings.end() && it->second.find(qname) != it->second.end()){
return true;
}
Binding::ptr bp = std::make_shared<Binding>(ename,qname,key);
if(durable){
if( _mapper.insert(bp) == false )return false;
}
auto &qmap = _bindings[ename];
qmap.insert(std::make_pair(qname,bp));
return true;
}
void unBind(const std::string &ename,const std::string &qname){
std::unique_lock<std::mutex> lock(_mutex);
auto eit = _bindings.find(ename);
if (eit == _bindings.end()) { return; }
auto qit = eit->second.find(qname);
if (qit == eit->second.end()) { return; }
_mapper.remove(ename, qname);
_bindings[ename].erase(qit);
}
void removeExchangeBindings(const std::string &ename){
std::unique_lock<std::mutex> lock(_mutex);
_mapper.removeExchangeBindings(ename);
_bindings.erase(ename);
}
void removeMsgqueueBindings(const std::string &qname){
std::unique_lock<std::mutex> lock(_mutex);
_mapper.removeMsgqueueBindings(qname);
for(auto&it :_bindings){
it.second.erase(qname);
}
}
MsgQueueBindMap getExchangeBindings(const std::string &ename){
std::unique_lock<std::mutex> lock(_mutex);
auto eit = _bindings.find(ename);
if(eit == _bindings.end()){
return MsgQueueBindMap();
}
return eit->second;
}
Binding::ptr getBinding(const std::string &ename, const std::string &qname){
std::unique_lock<std::mutex> lock(_mutex);
auto eit = _bindings.find(ename);
if(eit == _bindings.end()){
return Binding::ptr();
}
auto qit = eit->second.find(qname);
if(qit == eit->second.end()){
return Binding::ptr();
}
return qit->second;
}
bool exists(const std::string &ename, const std::string &qname) {
std::unique_lock<std::mutex> lock(_mutex);
auto eit = _bindings.find(ename);
if (eit == _bindings.end()) {
return false;
}
auto qit = eit->second.find(qname);
if (qit == eit->second.end()) {
return false;
}
return true;
}
size_t size() {
size_t total_size = 0;
std::unique_lock<std::mutex> lock(_mutex);
for (auto start = _bindings.begin(); start != _bindings.end(); ++start) {
total_size += start->second.size();
}
return total_size;
}
void clear() {
std::unique_lock<std::mutex> lock(_mutex);
_mapper.removeTable();
_bindings.clear();
}
private:
std::mutex _mutex;
BindingMapper _mapper;
BindingMap _bindings;
};
四、模块单元测试
对绑定模块进行全功能串联测试,覆盖绑定、解绑、查询、删除等逻辑。
cpp
#include "../mqserver/mq_binding.hpp"
#include <gtest/gtest.h>
Fy_mq::BindingManager::ptr bmp;
class BindingTest : public testing::Environment{
public:
virtual void SetUp() override {
bmp = std::make_shared<Fy_mq::BindingManager>("./data/meta.db");
}
virtual void TearDown() override {
bmp->clear();
}
};
// 1. 批量添加绑定 → 总数6
TEST(binding_test, step1_bind_6_items) {
bmp->bind("exchange1", "queue1", "news.music.#", true);
bmp->bind("exchange1", "queue2", "news.sport.#", true);
bmp->bind("exchange1", "queue3", "news.gossip.#", true);
bmp->bind("exchange2", "queue1", "news.music.pop", true);
bmp->bind("exchange2", "queue2", "news.sport.football", true);
bmp->bind("exchange2", "queue3", "news.gossip.#", true);
ASSERT_EQ(bmp->size(), 6);
}
// 2. 重复绑定 → 数量不变
TEST(binding_test, step2_rebind_same) {
bmp->bind("exchange1", "queue1", "news.music.#", true);
ASSERT_EQ(bmp->size(), 6);
}
// 3. 存在性判断
TEST(binding_test, step3_exists_check) {
ASSERT_TRUE(bmp->exists("exchange1", "queue1"));
ASSERT_TRUE(bmp->exists("exchange1", "queue2"));
ASSERT_TRUE(bmp->exists("exchange1", "queue3"));
ASSERT_TRUE(bmp->exists("exchange2", "queue1"));
ASSERT_TRUE(bmp->exists("exchange2", "queue2"));
ASSERT_TRUE(bmp->exists("exchange2", "queue3"));
ASSERT_FALSE(bmp->exists("no_ex", "queue1"));
ASSERT_FALSE(bmp->exists("exchange1", "no_queue"));
}
// 4. 获取单个绑定
TEST(binding_test, step4_get_binding) {
auto bp = bmp->getBinding("exchange1", "queue1");
ASSERT_NE(bp, nullptr);
ASSERT_EQ(bp->exchange_name, "exchange1");
ASSERT_EQ(bp->msgqueue_name, "queue1");
ASSERT_EQ(bp->binding_key, "news.music.#");
}
// 5. 获取交换机下所有绑定
TEST(binding_test, step5_get_exchange_bindings) {
auto bindings = bmp->getExchangeBindings("exchange1");
ASSERT_EQ(bindings.size(), 3);
auto bindings2 = bmp->getExchangeBindings("exchange2");
ASSERT_EQ(bindings2.size(), 3);
}
// 6. 解绑一个
TEST(binding_test, step6_unbind_one) {
bmp->unBind("exchange1", "queue1");
ASSERT_EQ(bmp->size(), 5);
ASSERT_FALSE(bmp->exists("exchange1", "queue1"));
}
// 7. 删除交换机下所有绑定
TEST(binding_test, step7_remove_exchange_bindings) {
bmp->removeExchangeBindings("exchange2");
ASSERT_EQ(bmp->size(), 2);
}
// 8. 删除队列所有绑定
TEST(binding_test, step8_remove_queue_bindings) {
bmp->removeMsgqueueBindings("queue3");
ASSERT_EQ(bmp->size(), 1);
}
// 9. 清空所有
TEST(binding_test, step9_clear_all) {
bmp->clear();
ASSERT_EQ(bmp->size(), 0);
}
int main(int argc,char *argv[])
{
testing::InitGoogleTest(&argc, argv);
testing::AddGlobalTestEnvironment(new BindingTest);
return RUN_ALL_TESTS();
}
五、功能说明
- 双层哈希结构 :
交换机名 → 队列名 → 绑定关系,便于快速查找与删除。 - 联合主键 :使用
交换机+队列作为联合主键,避免重复绑定。 - 线程安全:所有对外接口添加互斥锁,支持多线程并发访问。
- 持久化区分:持久化绑定写入数据库,非持久化绑定仅存在内存。
- 完整操作接口:支持绑定、解绑、按交换机删除、按队列删除、查询等功能。
- 数据恢复:服务启动时自动从数据库加载绑定关系。