RedisDB双机主从同步性能测试

安装redisDB

主节点

bash 复制代码

apt install redis-server

修改配置 /etc/redis/redis.conf

bash 复制代码

bind 0.0.0.0

save ""   # 禁止RDB持久化
#save 900 1
#save 300 10
#save 60 10000

appendonly no  # 禁止AOF持久化

重启服务

bash 复制代码

systemctl restart redis-server

从节点配置文件

bash 复制代码

bind 0.0.0.0

save ""   # 禁止RDB持久化
#save 900 1
#save 300 10
#save 60 10000

appendonly no  # 禁止AOF持久化

# 从节点关键配置
slaveof 172.40.20.132 6379

主从验证

主从状态

主节点执行 redis-cli info replication

bash 复制代码

root@r750-132:~# redis-cli  info replication
# Replication
role:master
connected_slaves:1
slave0:ip=172.40.20.131,port=6379,state=online,offset=84,lag=0
master_replid:1e214902bfe781c96a43a65ff20eefa9a9e6a75d
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:84
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:84

从节点检查

bash 复制代码

redis-cli info replication
# Replication
role:slave
master_host:172.40.20.132
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_repl_offset:168
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:1e214902bfe781c96a43a65ff20eefa9a9e6a75d
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:168
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:168

数据同步

主节点插入数据

bash 复制代码

root@r750-132:~# redis-cli set name1 ckun
OK
root@r750-132:~# redis-cli get name1
"ckun"

从节点读取数据

bash 复制代码

root@r750-131:~# redis-cli get name1
"ckun"

测试程序

安装hiredis API

bash 复制代码

git clone https://github.com/redis/hiredis.git
cd hiredis/
make
make install
ldconfig

测试代码

bash 复制代码

#include <stdio.h>
#include <stdint.h>
#include <pthread.h>
#include <signal.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <arpa/inet.h>
#include <sys/time.h>
#include <hiredis/hiredis.h>

typedef struct {
    uint32_t orig_saddr;
    uint32_t orig_daddr;
    uint8_t  proto;
    uint16_t orig_sport;
    uint16_t orig_dport;

    uint32_t new_saddr;
    uint32_t new_daddr;
    uint16_t new_sport;
    uint16_t new_dport;
} session_t;

#define CHANNEL "session_channel"
#define SESS_NUM 100000

uint16_t random_port() {
    return rand() % 65535 + 1;
}

char* session_to_string(session_t *session) {
    char *str = (char *)malloc(256);
    snprintf(str, 256,
             "{"
             "\"orig_saddr\":%u,"
             "\"orig_daddr\":%u,"
             "\"proto\":%u,"
             "\"orig_sport\":%u,"
             "\"orig_dport\":%u,"
             "\"new_saddr\":%u,"
             "\"new_daddr\":%u,"
             "\"new_sport\":%u,"
             "\"new_dport\":%u"
             "}",
             session->orig_saddr,
             session->orig_daddr,
             session->proto,
             session->orig_sport,
             session->orig_dport,
             session->new_saddr,
             session->new_daddr,
             session->new_sport,
             session->new_dport);
    return str;
}

static int count = 0;
static struct timeval start_time, end_time;
// 订阅消息的回调函数
void subscribe_callback(redisContext *c, redisReply *reply) {

    if (reply == NULL) {
        printf("Received NULL or empty message\n");
        return;
    }

    switch (reply->type) {
    case REDIS_REPLY_ERROR:
    case REDIS_REPLY_STRING:
    case REDIS_REPLY_NIL:
    case REDIS_REPLY_INTEGER:
    case REDIS_REPLY_STATUS:
        break;
    case REDIS_REPLY_ARRAY:
        {
            int i = 0;
            // for (i = 0; i < reply->elements; ++i)
            // {
            //     redisReply* childReply = reply->element[i];
            //     if (childReply->type == REDIS_REPLY_STRING)
            //     {
            //         printf("value:%s\n", childReply->str);
            //     }
            // }
            // 增加计数
            i = reply->elements - 1;
            redisReply* childReply = reply->element[i];
            if ( childReply->str != NULL) {
                //printf("value: %s\n", childReply->str);
                count++;
            }
        }
        break;
    default:
        break;
    }


    if (count == 1) {
        gettimeofday(&start_time, NULL); // 记录开始一时间
    }
    // 如果接收了 100,000 条数据，计算时间
    if (count == SESS_NUM) {
        gettimeofday(&end_time, NULL);
        double elapsed_time = (end_time.tv_sec - start_time.tv_sec) +
                (end_time.tv_usec - start_time.tv_usec) / 1000000.0;
        printf("Received %d sessions in %.3f seconds\n", SESS_NUM, elapsed_time);
        exit(0);
    }
}

// 订阅 Redis 频道并接收消息
void subscribe_to_redis(redisContext *c) {
    redisReply *reply;
    redisAppendCommand(c, "SUBSCRIBE %s", CHANNEL);
    while (1) {
        if (redisGetReply(c, (void **)&reply) == REDIS_OK) {
            // 调用回调函数处理数据
            subscribe_callback(c, reply);
            freeReplyObject(reply);
        }
    }
}

// 连接 Redis 并插入数据
void insert_sessions_to_redis(redisContext *c, int set_no_pub) {
    redisReply *reply = NULL;
    session_t session;
    char key[256];
    srand(time(NULL));
    gettimeofday(&start_time, NULL); // 记录开始一时间
    int printsize = 1;
    for (int i = 0; i < SESS_NUM; i++) {
        // 构造五元组 key
        session.orig_saddr = 0xC0A8000A + i;  // orig_saddr 递增
        session.orig_daddr = 0x0A000001;
        session.proto = 6;  // 假设是 TCP 协议
        session.orig_sport = 12345;
        session.orig_dport = 8080;

        // 为 session 生成随机数据
        session.new_saddr = 0xAC100001;
        session.new_daddr = 0xAC10000A;
        session.new_sport = random_port();
        session.new_dport = random_port();

        // 生成 Redis key (五元组)
        snprintf(key, sizeof(key), "SESS:%u:%u:%u:%u:%u", 
                 session.orig_saddr, session.orig_daddr, 
                 session.proto, session.orig_sport, session.orig_dport);

        // 将 session 转为字符串存入 Redis
        char *session_str = session_to_string(&session);

        if (printsize) {
            int valuesize = strlen(session_str);
            int keysize = strlen(key);
            printf("KeySize: %d, ValueSize: %d\n", keysize, valuesize);
            printsize = 0;
        }
        reply = redisCommand(c, "SET %s %s", key, session_str);
        free(session_str);

        if (reply != NULL) {
            freeReplyObject(reply);
        }

        // 发布消息到订阅频道
        if (set_no_pub == 0) {
            redisCommand(c, "PUBLISH %s %s", CHANNEL, key);  // 发布消息
        }
    }
    gettimeofday(&end_time, NULL);
    double elapsed_time = (end_time.tv_sec - start_time.tv_sec) +
            (end_time.tv_usec - start_time.tv_usec) / 1000000.0;
    printf("SET %d sessions in %.3f seconds\n", SESS_NUM, elapsed_time);
}


int sub_session() {
    redisContext *c = redisConnect("127.0.0.1", 6379);
    if (c == NULL || c->err) {
        if (c) {
            printf("Redis connection error: %s\n", c->errstr);
        } else {
            printf("Redis connection error: can't allocate redis context\n");
        }
        return 1;
    }

    // 订阅 Redis 频道
    subscribe_to_redis(c);

    redisFree(c);
    return 0;
}

int pub_session(int set_no_pub) {
    redisContext *c = redisConnect("127.0.0.1", 6379);
    if (c == NULL || c->err) {
        if (c) {
            printf("Redis connection error: %s\n", c->errstr);
        } else {
            printf("Redis connection error: can't allocate redis context\n");
        }
        return 1;
    }
    printf("Begin to insert data\n");
    // 插入数据
    insert_sessions_to_redis(c, set_no_pub);

    redisFree(c);
    return 0;
}

int main(int argc, char* argv[])
{
    if (argc < 2) {
        printf("Usage: ./redistest pub / sub\n");
        return 0;
    }

    if(strlen(argv[1]) >= 3) {
        if(strcmp(argv[1], "pub") == 0) {
            return pub_session(0);
        } else if (strcmp(argv[1], "set_no_pub") == 0) {
            return pub_session(1);
        }   else if (strcmp(argv[1], "sub") == 0) {
            return sub_session();
        }
    } 

    return 0;
}

Makefile

bash 复制代码

APP_tool = redistest

SRCS-t = $(wildcard *.c) 
PKGCONF ?= pkg-config

CFLAGS += -g -O2 $(shell $(PKGCONF) --cflags hiredis) 
LDFLAGS += $(shell $(PKGCONF) --libs hiredis) -lpthread 

all: $(APP_tool)

$(APP_tool): $(SRCS-t)
	$(CC) $(CFLAGS) $(SRCS-t) -o $@ $(LDFLAGS)

.PHONY: clean
clean:
	rm $(APP_tool)

RedisDB命令

bash 复制代码

# 清空数据库
redis-cli FLUSHDB  

# 找到 SESS:前缀的 KEY， 通过这些KEY可以查到到VALUE
ckun@r750-131:~$ redis-cli KEYS "SESS:*"
 1) "SESS:3232235530:167772161:6:12345:8080"
 2) "SESS:3232235537:167772161:6:12345:8080"
 3) "SESS:3232235538:167772161:6:12345:8080"
 4) "SESS:3232235539:167772161:6:12345:8080"
 5) "SESS:3232235531:167772161:6:12345:8080"
 6) "SESS:3232235532:167772161:6:12345:8080"
 7) "SESS:3232235533:167772161:6:12345:8080"
 8) "SESS:3232235536:167772161:6:12345:8080"
 9) "SESS:3232235535:167772161:6:12345:8080"
10) "SESS:3232235534:167772161:6:12345:8080"

# 通过KEY 查询VALUE
ckun@r750-131:~$ redis-cli get "SESS:3232235530:167772161:6:12345:8080"
"{\"orig_saddr\":3232235530,\"orig_daddr\":167772161,\"proto\":6,\"orig_sport\":12345,\"orig_dport\":8080,\"new_saddr\":2886729729,\"new_daddr\":2886729738,\"new_sport\":3543,\"new_dport\":54296}"

测试

主写入10万条记录，这里以NAT会话作为写入条目，KEY 38字节，VALUE 177字节。

从订阅10万条记录，看看同步一共花费多少时间。

主设备写入10万条花了9.49秒

bash 复制代码

ckun@r750-132:~/ws/redistest$ ./redistest pub 
Begin to insert data
KeySize: 38, ValueSize: 177
SET 100000 sessions in 9.493 seconds

从设备同步时间也是一致的（从订阅到第一个记录开始计时）

bash 复制代码

ckun@r750-131:~$ ./redistest sub
Received 100000 sessions in 9.493 seconds

平均一秒写入并同步一万条。

那只写入，不PUB看一下性能

bash 复制代码

ckun@r750-132:~/ws/redistest$ ./redistest set_no_pub
Begin to insert data
KeySize: 38, ValueSize: 177
SET 100000 sessions in 4.484 seconds

平均有20000多条了，看起来加上PUB是要多消耗一倍的计算资源。

单机写入Benchmark

bash 复制代码

ckun@r750-132:~/ws/redistest$ redis-benchmark -h  127.0.0.1 -p 6379 -c 8 -n 100000  -t set -d 100 -P 8 -q 
SET: 395288.56 requests per second

ckun@r750-132:~/ws/redistest$ redis-benchmark -h  127.0.0.1 -p 6379 -c 8 -n 100000  -t set -d 100 -P 1 -q
SET: 134408.59 requests per second

ckun@r750-132:~/ws/redistest$ redis-benchmark -h  127.0.0.1 -p 6379 -c 1 -n 100000  -t set -d 100 -P 1 -q 
SET: 18419.60 requests per second

单连接，一个请求写入一个记录，性能18000 req/s 。

小结

在当前Intel® Xeon® Gold 6346 CPU @ 3.10GHz 环境下，

测试程序单连接写入并且同步只有 10000 req/s 左右的写入性能。

如果关闭PUB，单纯写入会有20000 req/s左右性能。

通过benchmark验证，单链接也有18000 req/s左右写入性能。

在大容量大并发的NAT环境中，20W每秒的新建会话速度，每个会话得有两个表，上下行各一个，总计40W个条目，40个线程并发写入才能满足。这明显不能满足性能需求。