仿RabbitMQ实现消息队列-服务端核心模块实现(2)

队列数据管理模块

在消息队列系统中,队列作为消息存储与转发的核心单元,其元数据管理与持久化是保证服务可靠性的关键。本篇介绍消息队列中间件中队列数据的内存管理、持久化存储、线程安全处理完整实现,代码结构与交换机数据管理保持统一,具备工业级可用性。

一、队列结构体定义

队列结构体用于封装队列核心属性,包含名称、持久化、排他、自动删除及扩展参数,并提供参数序列化与反序列化接口,便于数据库存储与读取。

cpp 复制代码
struct MsgQueue{
    using ptr = std::shared_ptr<MsgQueue>;

    std::string name;
    bool durable;
    bool exclusive;
    bool auto_delete;
    google::protobuf::Map<std::string,std::string> args;

    MsgQueue() = default;
    MsgQueue(const std::string& qname,
            bool qdurable,
            bool qexclusive,
            bool qauto_delete,
            const google::protobuf::Map<std::string, std::string>& qargs)
        : name(qname),
        durable(qdurable),
        exclusive(qexclusive),
        auto_delete(qauto_delete),
        args(qargs) {}
    
    // 字符串参数解析到Map
    void SetArgs(const std::string &str_args){
        std::vector<std::string> sub_args;
        StrHelper::split(str_args,"&",sub_args);
        for(auto &str : sub_args){
            size_t pos = str.find("=");
            if( pos == std::string::npos){
                ELOG("交换机解析格式非法:%s",str.c_str());
                continue;
            }
            std::string key = str.substr(0,pos);
            std::string value = str.substr(pos+1);  
            args[key] = value;
        }
    }

    // Map参数序列化为字符串
    std::string getArgs() const {
        std::string result;
        for(auto &it:args){
            result += it.first + "=" + it.second + "&";
        }
        return result;
    }
};

二、队列数据持久化实现

基于SQLite实现队列元数据持久化,负责队列数据的增、删、查、表创建与数据恢复,是服务重启后数据不丢失的核心。

cpp 复制代码
using QueueMap = std::unordered_map<std::string,MsgQueue::ptr>;
class MsgQueueMapper{
public:
    MsgQueueMapper(const std::string &dbfile) : _sql_helper(dbfile){
        std::string path = FileHelper::parentDirectory(dbfile);
        FileHelper::createDirectory(path);
        _sql_helper.open();
        createTable();
    }

    // 创建队列数据表
    void createTable(){
        std::stringstream sql;
        sql << "create table if not exists queue_table(";
        sql << "name varchar(32) primary key,";
        sql << "durable int,";
        sql << "exclusive int,";
        sql << "auto_delete int,";
        sql << "args varchar(128));";
        assert(_sql_helper.exec(sql.str(),nullptr,nullptr));
    }

    // 删除数据表
    void removeTable(){
        std::string sql = "drop table if exists queue_table";
        _sql_helper.exec(sql,nullptr,nullptr);
    }
    
    // 插入队列数据
    bool insert(MsgQueue::ptr &queue){
        std::stringstream sql;
        sql << "insert into queue_table(name,durable,exclusive,auto_delete,args) values(";
        sql << "'" << queue->name <<"',";
        sql << (int)queue->durable <<",";
        sql << (int)queue->exclusive <<",";
        sql << (int)queue->auto_delete <<",";
        sql << "'" << queue->getArgs() <<"');";
        return _sql_helper.exec(sql.str(),nullptr,nullptr);
    }

    // 删除指定队列数据
    void remove(const std::string &name){
        std::string sql = "delete from queue_table where name = '" + name + "'";
        _sql_helper.exec(sql,nullptr,nullptr);
    }

    // 从数据库恢复所有队列数据
    QueueMap recovery() {
        QueueMap result;
        std::string sql = "select name,durable,exclusive,auto_delete,args from queue_table;";
        _sql_helper.exec(sql,selectCallback,&result);
        return result;
    }
private:
    // 数据读取回调函数
    static int selectCallback(void* arg,int numcol,char** row,char** fields){
        QueueMap *result = (QueueMap *)arg;
        if(!row[0] || !row[1] || !row[2] || !row[3]) return 0;
        MsgQueue::ptr mqp = std::make_shared<MsgQueue>();
        mqp->name = row[0];
        mqp->durable = (bool)std::stoi(row[1]);
        mqp->exclusive = (bool)std::stoi(row[2]);
        mqp->auto_delete = (bool)std::stoi(row[3]);
        if(row[4]) mqp->SetArgs(row[4]);
        result->insert(std::make_pair(mqp->name,mqp));
        return 0;
    }
private:
    SqliteHelper _sql_helper;
};

三、线程安全的队列管理类

对外提供统一的队列操作接口,内部维护内存队列映射,所有公共接口加锁保证多线程安全,区分持久化与非持久化队列逻辑。

cpp 复制代码
class MsgQueueManager {
public:
    using ptr = std::shared_ptr<MsgQueueManager>;

    MsgQueueManager(const std::string &dbfile):_mapper(dbfile) {
        std::unique_lock<std::mutex> lock(_mutex);
        _msg_queues = _mapper.recovery();
    }

    // 声明队列:存在则返回true,不存在则创建
    bool declareQueue(const std::string &qname,
        bool qdurable,
        bool qexclusive,
        bool qauto_delete,
        const google::protobuf::Map<std::string,std::string> &qargs){
        std::unique_lock<std::mutex> lock(_mutex);
        auto it = _msg_queues.find(qname);
        if( it != _msg_queues.end()){
            return true;
        }
        MsgQueue::ptr mqp = std::make_shared<MsgQueue>();
        mqp->name = qname;
        mqp->durable = qdurable;
        mqp->exclusive = qexclusive;
        mqp->auto_delete = qauto_delete;
        mqp->args = qargs;
        if(qdurable == true){
            if(!_mapper.insert(mqp)){
                return false;
            }
        }
        _msg_queues.insert(std::make_pair(qname,mqp));
        return true;
    }

    // 删除队列:持久化队列同步删除数据库数据
    void deleteQueue(const std::string &name){
        std::unique_lock<std::mutex> lock(_mutex);
        auto it = _msg_queues.find(name);
        if( it == _msg_queues.end()) return;
        if( it->second->durable == true){
            _mapper.remove(name);
        }
        _msg_queues.erase(it);
    }

    // 查询指定队列
    MsgQueue::ptr selectQueue(const std::string &name){
        std::unique_lock<std::mutex> lock(_mutex);
        auto it = _msg_queues.find(name);
        if( it == _msg_queues.end()) return MsgQueue::ptr();
        return it->second;
    }

    // 获取所有队列
    QueueMap allQueues() {
        std::unique_lock<std::mutex> lock(_mutex);
        return _msg_queues;
    }

    // 判断队列是否存在
    bool exists(const std::string &name){
        std::unique_lock<std::mutex> lock(_mutex);
        auto it = _msg_queues.find(name);
        if(it == _msg::end()) return false;
        return true;
    }
    
    // 获取队列数量
    size_t size(){
        std::unique_lock<std::mutex> lock(_mutex);
        return _msg_queues.size();
    }

    // 清空所有队列与数据表
    void clear(){
        std::unique_lock<std::mutex> lock(_mutex);
        _mapper.removeTable();
        _msg_queues.clear();
    }
private:
    std::mutex _mutex;
    MsgQueueMapper _mapper;
    QueueMap _msg_queues;
};

四、模块单元测试(GTest)

对队列管理模块做全功能串联测试,覆盖声明、删除、查询、存在判断、非持久化验证、清空等逻辑。

cpp 复制代码
#include "../mqserver/mq_queue.hpp"
#include <gtest/gtest.h>
#include <vector>

Fy_mq::MsgQueueManager::ptr mqmp;

class QueueTest : public testing::Environment
{
public:
    virtual void SetUp() override
    {
        static std::once_flag flag;
        std::call_once(flag, [&]() {
            mqmp = std::make_shared<Fy_mq::MsgQueueManager>("./data/meta.db");
        });
    }

    virtual void TearDown() override
    {
        mqmp->clear();
    }
};

// 1. 声明4个队列
TEST(queue_test, step1_declare_4_queues) {
    google::protobuf::Map<std::string, std::string> args;
    args["k1"] = "v1";

    mqmp->declareQueue("queue1", true, false, false, args);
    mqmp->declareQueue("queue2", true, false, false, args);
    mqmp->declareQueue("queue3", true, false, false, args);
    mqmp->declareQueue("queue4", true, false, false, args);

    ASSERT_EQ(mqmp->size(), 4);
}

// 2. 重复声明队列
TEST(queue_test, step2_duplicate_declare) {
    google::protobuf::Map<std::string, std::string> args;
    bool ret = mqmp->declareQueue("queue1", true, false, false, args);

    ASSERT_TRUE(ret);
    ASSERT_EQ(mqmp->size(), 4);
}

// 3. 存在判断与查询
TEST(queue_test, step3_exists_and_select) {
    ASSERT_TRUE(mqmp->exists("queue1"));
    ASSERT_TRUE(mqmp->exists("queue2"));
    ASSERT_FALSE(mqmp->exists("queue_not_exist"));

    auto q = mqmp->selectQueue("queue3");
    ASSERT_NE(q, nullptr);
    ASSERT_EQ(q->name, "queue3");
}

// 4. 删除队列
TEST(queue_test, step4_delete_2_queues) {
    mqmp->deleteQueue("queue1");
    mqmp->deleteQueue("queue2");

    ASSERT_EQ(mqmp->size(), 2);
    ASSERT_FALSE(mqmp->exists("queue1"));
    ASSERT_FALSE(mqmp->exists("queue2"));
}

// 5. 获取所有队列
TEST(queue_test, step5_all_queues) {
    auto queues = mqmp->allQueues();
    ASSERT_EQ(queues.size(), 2);
}

// 6. 非持久化队列不恢复
TEST(queue_test, step7_non_durable_no_recovery) {
    mqmp->clear();

    google::protobuf::Map<std::string, std::string> args;
    mqmp->declareQueue("temp_queue", false, false, false, args);

    mqmp.reset();
    mqmp = std::make_shared<Fy_mq::MsgQueueManager>("./data/meta.db");

    ASSERT_FALSE(mqmp->exists("temp_queue"));
}

// 7. 清空所有数据
TEST(queue_test, step8_clear_all) {
    mqmp->clear();
    ASSERT_EQ(mqmp->size(), 0);
}

int main(int argc,char *argv[]){
    testing::InitGoogleTest(&argc,argv);
    testing::AddGlobalTestEnvironment(new QueueTest);
    return RUN_ALL_TESTS();
}

运行结果:

bash 复制代码
zhangwho@VM-0-2-ubuntu:~/mq/mqtest$ ./mq_queuetest 
[==========] Running 7 tests from 1 test suite.
[----------] Global test environment set-up.
[----------] 7 tests from queue_test
[ RUN      ] queue_test.step1_declare_4_queues
[       OK ] queue_test.step1_declare_4_queues (23 ms)
[ RUN      ] queue_test.step2_duplicate_declare
[       OK ] queue_test.step2_duplicate_declare (0 ms)
[ RUN      ] queue_test.step3_exists_and_select
[       OK ] queue_test.step3_exists_and_select (0 ms)
[ RUN      ] queue_test.step4_delete_2_queues
[       OK ] queue_test.step4_delete_2_queues (12 ms)
[ RUN      ] queue_test.step5_all_queues
[       OK ] queue_test.step5_all_queues (0 ms)
[ RUN      ] queue_test.step7_non_durable_no_recovery
[       OK ] queue_test.step7_non_durable_no_recovery (12 ms)
[ RUN      ] queue_test.step8_clear_all
[       OK ] queue_test.step8_clear_all (4 ms)
[----------] 7 tests from queue_test (51 ms total)

[----------] Global test environment tear-down
[==========] 7 tests from 1 test suite ran. (58 ms total)
[  PASSED  ] 7 tests.

五、功能说明

  1. 持久化区分:持久化队列写入SQLite,重启自动恢复;非持久化队列仅存内存,重启消失。
  2. 线程安全:所有对外接口使用互斥锁保护,支持多线程并发访问。
  3. 幂等声明:同一队列多次声明不会重复创建,符合消息队列语义。
  4. 数据恢复:服务启动时自动从数据库加载队列数据。
  5. 完整接口:支持声明、删除、查询、判断存在、获取全部、清空等操作。

绑定数据管理模块

绑定是交换机与队列之间的关联关系,通过绑定键实现消息的路由转发。为了保证服务重启之后绑定关系不丢失,需要对绑定数据进行持久化管理。本篇文章介绍绑定数据结构定义、数据持久化、线程安全管理的完整实现,代码结构与交换机、队列模块保持一致。

一、绑定结构体定义

用于描述交换机与队列之间的绑定关系,包含交换机名称、队列名称、绑定键三个核心属性。

cpp 复制代码
struct Binding{
    using ptr = std::shared_ptr<Binding>;
    std::string exchange_name;
    std::string msgqueue_name;
    std::string binding_key;

    Binding() = default;
    Binding(const std::string &ename,const std::string &qname,const std::string &key)
        :exchange_name(ename),msgqueue_name(qname),binding_key(key){}
};

using MsgQueueBindMap = std::unordered_map<std::string,Binding::ptr>;
using BindingMap = std::unordered_map<std::string,MsgQueueBindMap>;

二、绑定数据持久化实现

基于 SQLite 实现绑定数据的持久化存储,提供绑定新增、删除、按交换机删除、按队列删除、数据恢复等功能。

cpp 复制代码
class BindingMapper{
public:
    BindingMapper(const std::string &dbfile): _sql_helper(dbfile){
        std::string path = FileHelper::parentDirectory(dbfile);
        FileHelper::createDirectory(path);
        _sql_helper.open();
        createTable();
    }

    void createTable(){
        std::stringstream sql;
        sql << "create table if not exists binding_table(";
        sql << "exchange_name varchar(32), ";
        sql << "msgqueue_name varchar(32), ";
        sql << "binding_key varchar(128),";
        sql << "primary key(exchange_name, msgqueue_name));";
        assert(_sql_helper.exec(sql.str(), nullptr, nullptr));
    }

    void removeTable(){
        std::string sql = "drop table if exists binding_table;";
        _sql_helper.exec(sql, nullptr, nullptr);
    }

    bool insert(Binding::ptr &binding){
        std::stringstream sql;
        sql << "insert into binding_table(exchange_name,msgqueue_name,binding_key) values(";
        sql << "'" << binding->exchange_name << "',";
        sql << "'" << binding->msgqueue_name << "',";
        sql << "'" << binding->binding_key << "');";
        return _sql_helper.exec(sql.str(),nullptr,nullptr);
    }

    void remove(const std::string &ename,const std::string &qname){
        std::stringstream sql;
        sql << "delete from binding_table where ";
        sql << "exchange_name = '" << ename << "' and ";
        sql << "msgqueue_name = '" << qname << "';";
        _sql_helper.exec(sql.str(),nullptr,nullptr);
    }

    void removeExchangeBindings(const std::string &ename){
        std::stringstream sql;
        sql << "delete from binding_table where ";
        sql << "exchange_name = '" << ename << "';";
        _sql_helper.exec(sql.str(),nullptr,nullptr);
    }

    void removeMsgqueueBindings(const std::string &qname){
        std::stringstream sql;
        sql << "delete from binding_table where ";
        sql << "msgqueue_name = '" << qname << "';";
        _sql_helper.exec(sql.str(),nullptr,nullptr);
    }

    BindingMap recovery() {
        BindingMap result;
        std::string sql = "select exchange_name , msgqueue_name , binding_key from binding_table;";
        _sql_helper.exec(sql,selectCallback,&result);
        return result;
    }
private:
    static int selectCallback(void* arg,int numcol,char** row,char** fields){
        if (!row[0] || !row[1] || !row[2]) return 0;
        BindingMap *result = (BindingMap *)arg;
        auto& qmap = (*result)[row[0]];
        Binding::ptr bdp = std::make_shared<Binding>();
        bdp->exchange_name = row[0];
        bdp->msgqueue_name = row[1];
        bdp->binding_key = row[2];
        qmap.insert(std::make_pair(bdp->msgqueue_name,bdp));
        return 0;
    }
private:
    SqliteHelper _sql_helper;
};

三、线程安全绑定管理类

对外提供统一的绑定操作接口,所有操作均加锁保证线程安全,区分持久化与非持久化绑定逻辑。

cpp 复制代码
class BindingManager {
public:
    using ptr = std::shared_ptr<BindingManager>;

    BindingManager(const std::string &dbfile):_mapper(dbfile){
        std::unique_lock<std::mutex> lock(_mutex);
        _bindings = _mapper.recovery();
    }
    
    bool bind(const std::string &ename,const std::string &qname,const std::string &key,bool durable){
        std::unique_lock<std::mutex> lock(_mutex);
        auto it = _bindings.find(ename);
        if(it != _bindings.end() && it->second.find(qname) != it->second.end()){
            return true;
        }
        Binding::ptr bp = std::make_shared<Binding>(ename,qname,key);
        if(durable){
            if( _mapper.insert(bp) == false )return false;
        }
        auto &qmap = _bindings[ename];
        qmap.insert(std::make_pair(qname,bp));
        return true;
    }

    void unBind(const std::string &ename,const std::string &qname){
        std::unique_lock<std::mutex> lock(_mutex);
        auto eit = _bindings.find(ename);
        if (eit == _bindings.end()) { return; }
        auto qit = eit->second.find(qname);
        if (qit == eit->second.end()) { return; }
        _mapper.remove(ename, qname);
        _bindings[ename].erase(qit);
    }

    void removeExchangeBindings(const std::string &ename){
        std::unique_lock<std::mutex> lock(_mutex);
        _mapper.removeExchangeBindings(ename);
        _bindings.erase(ename);
    }

    void removeMsgqueueBindings(const std::string &qname){
        std::unique_lock<std::mutex> lock(_mutex);
        _mapper.removeMsgqueueBindings(qname);
        for(auto&it :_bindings){
            it.second.erase(qname);
        }
    }

    MsgQueueBindMap getExchangeBindings(const std::string &ename){
        std::unique_lock<std::mutex> lock(_mutex);
        auto eit = _bindings.find(ename);
        if(eit == _bindings.end()){
            return MsgQueueBindMap();
        }
        return eit->second;
    }

    Binding::ptr getBinding(const std::string &ename, const std::string &qname){
        std::unique_lock<std::mutex> lock(_mutex);
        auto eit = _bindings.find(ename);
        if(eit == _bindings.end()){
            return Binding::ptr();
        }
        auto qit = eit->second.find(qname);
        if(qit == eit->second.end()){
            return Binding::ptr();
        }
        return qit->second;
    }

    bool exists(const std::string &ename, const std::string &qname) {
        std::unique_lock<std::mutex> lock(_mutex);
        auto eit = _bindings.find(ename);
        if (eit == _bindings.end()) { 
            return false; 
        }
        auto qit = eit->second.find(qname);
        if (qit == eit->second.end()) { 
            return false; 
        }
        return true;
    }

    size_t size() {
        size_t total_size = 0;
        std::unique_lock<std::mutex> lock(_mutex);
        for (auto start = _bindings.begin(); start != _bindings.end(); ++start) {
            total_size += start->second.size();
        }
        return total_size;
    }

    void clear() {
        std::unique_lock<std::mutex> lock(_mutex);
        _mapper.removeTable();
        _bindings.clear();
    }
    
private:
    std::mutex _mutex;
    BindingMapper _mapper;
    BindingMap _bindings;
};

四、模块单元测试

对绑定模块进行全功能串联测试,覆盖绑定、解绑、查询、删除等逻辑。

cpp 复制代码
#include "../mqserver/mq_binding.hpp"
#include <gtest/gtest.h>

Fy_mq::BindingManager::ptr bmp;

class BindingTest : public testing::Environment{
public:
    virtual void SetUp() override {
        bmp = std::make_shared<Fy_mq::BindingManager>("./data/meta.db");
    }
    virtual void TearDown() override {
        bmp->clear();
    }
};

// 1. 批量添加绑定 → 总数6
TEST(binding_test, step1_bind_6_items) {
    bmp->bind("exchange1", "queue1", "news.music.#", true);
    bmp->bind("exchange1", "queue2", "news.sport.#", true);
    bmp->bind("exchange1", "queue3", "news.gossip.#", true);
    bmp->bind("exchange2", "queue1", "news.music.pop", true);
    bmp->bind("exchange2", "queue2", "news.sport.football", true);
    bmp->bind("exchange2", "queue3", "news.gossip.#", true);
    ASSERT_EQ(bmp->size(), 6);
}

// 2. 重复绑定 → 数量不变
TEST(binding_test, step2_rebind_same) {
    bmp->bind("exchange1", "queue1", "news.music.#", true);
    ASSERT_EQ(bmp->size(), 6);
}

// 3. 存在性判断
TEST(binding_test, step3_exists_check) {
    ASSERT_TRUE(bmp->exists("exchange1", "queue1"));
    ASSERT_TRUE(bmp->exists("exchange1", "queue2"));
    ASSERT_TRUE(bmp->exists("exchange1", "queue3"));
    ASSERT_TRUE(bmp->exists("exchange2", "queue1"));
    ASSERT_TRUE(bmp->exists("exchange2", "queue2"));
    ASSERT_TRUE(bmp->exists("exchange2", "queue3"));

    ASSERT_FALSE(bmp->exists("no_ex", "queue1"));
    ASSERT_FALSE(bmp->exists("exchange1", "no_queue"));
}

// 4. 获取单个绑定
TEST(binding_test, step4_get_binding) {
    auto bp = bmp->getBinding("exchange1", "queue1");
    ASSERT_NE(bp, nullptr);
    ASSERT_EQ(bp->exchange_name, "exchange1");
    ASSERT_EQ(bp->msgqueue_name, "queue1");
    ASSERT_EQ(bp->binding_key, "news.music.#");
}

// 5. 获取交换机下所有绑定
TEST(binding_test, step5_get_exchange_bindings) {
    auto bindings = bmp->getExchangeBindings("exchange1");
    ASSERT_EQ(bindings.size(), 3);

    auto bindings2 = bmp->getExchangeBindings("exchange2");
    ASSERT_EQ(bindings2.size(), 3);
}

// 6. 解绑一个
TEST(binding_test, step6_unbind_one) {
    bmp->unBind("exchange1", "queue1");
    ASSERT_EQ(bmp->size(), 5);
    ASSERT_FALSE(bmp->exists("exchange1", "queue1"));
}

// 7. 删除交换机下所有绑定
TEST(binding_test, step7_remove_exchange_bindings) {
    bmp->removeExchangeBindings("exchange2");
    ASSERT_EQ(bmp->size(), 2);
}

// 8. 删除队列所有绑定
TEST(binding_test, step8_remove_queue_bindings) {
    bmp->removeMsgqueueBindings("queue3");
    ASSERT_EQ(bmp->size(), 1);
}

// 9. 清空所有
TEST(binding_test, step9_clear_all) {
    bmp->clear();
    ASSERT_EQ(bmp->size(), 0);
}

int main(int argc,char *argv[])
{
    testing::InitGoogleTest(&argc, argv);
    testing::AddGlobalTestEnvironment(new BindingTest);
    return RUN_ALL_TESTS();
}

五、功能说明

  1. 双层哈希结构交换机名 → 队列名 → 绑定关系,便于快速查找与删除。
  2. 联合主键 :使用交换机+队列作为联合主键,避免重复绑定。
  3. 线程安全:所有对外接口添加互斥锁,支持多线程并发访问。
  4. 持久化区分:持久化绑定写入数据库,非持久化绑定仅存在内存。
  5. 完整操作接口:支持绑定、解绑、按交换机删除、按队列删除、查询等功能。
  6. 数据恢复:服务启动时自动从数据库加载绑定关系。
相关推荐
Hello_Embed1 小时前
【无标题】
网络·笔记·网络协议·tcp/ip·嵌入式
路溪非溪2 小时前
详解下DNS协议
网络·网络协议·tcp/ip·智能路由器
CHANG_THE_WORLD2 小时前
<Fluent Python > 2. 第二章:序列的数组
网络·windows·python
byoass2 小时前
企业云盘API集成指南:如何与CI/CD流水线打通
网络·安全·ci/cd·云计算
大卡片2 小时前
TCP、IP和TFTP协议
服务器·网络·tcp/ip
汽车仪器仪表相关领域2 小时前
Kvaser Memorator Professional HS/LS:高速 + 低速双通道 CAN 总线记录仪,跨系统诊断的专业级解决方案
网络·人工智能·功能测试·测试工具·安全·压力测试
志栋智能3 小时前
跨越人机边界:超自动化巡检如何重塑工作流程?
运维·网络·人工智能·安全·自动化
仍然.3 小时前
网络编程(一)---传输层协议和UDP数据报套接字编程
网络·网络协议·udp
zzipeng3 小时前
IMX6ULL CAN通讯应用学习
linux·运维·网络