Redis系列之keys命令和scan命令性能对比

smileNicky2023-12-06 14:21

项目场景

Redis的keys *命令在生产环境是慎用的，特别是一些并发量很大的项目，原因是Redis是单线程的，keys *会引发Redis锁，占用reids CPU，如果key数量很大而且并发是比较大的情况，效率是很慢的，很有可能导致服务雪崩，在Redis官方的文档是这样解释的，官方的推荐是使用scan命令或者集合

解决方案

搭建一个工程来实践一下，项目环境：

  • JDK 1.8

  • SpringBoot 2.2.1

  • Maven 3.2+

  • Mysql 8.0.26

  • spring-boot-starter-data-redis 2.2.1

  • jedis3.1.0

  • 开发工具

    • IntelliJ IDEA

    • smartGit

新建一个SpringBoot项目

选择需要的依赖

选择Maven项目和jdk对应的版本

xml 复制代码 
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>2.2.1.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>
    <groupId>com.example</groupId>
    <artifactId>springboot-jedis</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>springboot-jedis</name>
    <description>Demo project for Spring Boot</description>
    <properties>
        <java.version>8</java.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-data-redis</artifactId>
            <exclusions>
                <exclusion>
                    <groupId>io.lettuce</groupId>
                    <artifactId>lettuce-core</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>redis.clients</groupId>
            <artifactId>jedis</artifactId>
        </dependency>

        <dependency>
            <groupId>cn.hutool</groupId>
            <artifactId>hutool-all</artifactId>
            <version>5.7.11</version>
        </dependency>

        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
                <configuration>
                    <excludes>
                        <exclude>
                            <groupId>org.projectlombok</groupId>
                            <artifactId>lombok</artifactId>
                        </exclude>
                    </excludes>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>
java 复制代码 
package com.example.jedis.configuration;

import com.example.jedis.common.JedisTemplate;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.jedis.JedisConnection;
import org.springframework.data.redis.connection.jedis.JedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.repository.configuration.EnableRedisRepositories;
import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;

@Configuration
@ConditionalOnClass({GenericObjectPool.class, JedisConnection.class, Jedis.class})
@EnableRedisRepositories(basePackages = "com.example.jedis.repository")
@Slf4j
public class RedisConfiguration {


    @Bean
    public JedisPoolConfig jedisPoolConfig() {
        return new JedisPoolConfig();
    }

    @Bean
    public JedisPool jedisPool() {
      return new JedisPool(jedisPoolConfig());
    }


    @Bean
    public RedisConnectionFactory jedisConnectionFactory() {
        return new JedisConnectionFactory();
    }

    @Bean
    public RedisTemplate<String, Object> redisTemplate() {
        RedisTemplate<String, Object> template = new RedisTemplate<String, Object>();
        template.setConnectionFactory(jedisConnectionFactory());
        template.setKeySerializer(new StringRedisSerializer());
        template.setValueSerializer(new GenericJackson2JsonRedisSerializer());
        return template;
    }

    




}

写一个工具类，实现redis scan和keys *的逻辑，当然也可以直接使用RedisTemplate

java 复制代码 
import cn.hutool.core.collection.ConcurrentHashSet;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.stereotype.Component;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import redis.clients.jedis.exceptions.JedisConnectionException;
import redis.clients.jedis.exceptions.JedisDataException;
import redis.clients.jedis.exceptions.JedisException;
import redis.clients.jedis.params.SetParams;

import javax.annotation.Resource;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;
@Component
@Slf4j
public class JedisUtil implements InitializingBean {


    @Resource
    private JedisPool jedisPool;

    private Jedis jedis;

    public JedisTemplate(JedisPool jedisPool) {
        this.jedisPool = jedisPool;
    }

    public JedisTemplate() {

    }

    @Override
    public void afterPropertiesSet() {
        jedis = jedisPool.getResource();
    }


    public <T> T execute(Function<Jedis, T> action) {
        T apply = null;
        try {
            jedis = jedisPool.getResource();
            apply = action.apply(jedis);
        } catch (JedisException e) {
            handleException(e);
            throw e;
        } finally {
            jedis.close();
        }
        return apply;
    }

    public void execute(Consumer<Jedis> action) {
        try {
            jedis = jedisPool.getResource();
            action.accept(jedis);
        } catch (JedisException e) {
            handleException(e);
            throw e;
        } finally {
            jedis.close();
        }
    }

    public JedisPool getJedisPool() {
        return this.jedisPool;
    }
	
	 public Set<String> keys(final String pattern) {
        return execute(e->{
            return jedis.keys(pattern);
        });
    }

    public Set<String> scan(String pattern) {
        return execute(e->{
            return this.doScan(pattern);
        });
    }


    protected Set<String> doScan(String pattern) {
        Set<String> resultSet = new ConcurrentHashSet<>();
        String cursor = String.valueOf(0);
        try {
            do {
                ScanParams params = new ScanParams();
                params.count(300);
                params.match(pattern);
                ScanResult<String> scanResult = jedis.scan(cursor, params);
                cursor = scanResult.getCursor();
                resultSet.addAll(scanResult.getResult());
            } while (Integer.valueOf(cursor) > 0);
        } catch (NumberFormatException e) {
            log.error("doScan NumberFormatException:{}", e);
        } catch (Exception e) {
            log.error("doScan Exception :{}", e);
        }
        return resultSet;
    }


    protected void handleException(JedisException e) {
        if (e instanceof JedisConnectionException) {
            log.error("redis connection exception:{}", e);
        } else if (e instanceof JedisDataException) {
            log.error("jedis data exception:{}", e);
        } else {
            log.error("jedis exception:{}", e);
        }
    }

}

新增测试类

java 复制代码 
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.TimeInterval;
import cn.hutool.core.thread.ExecutorBuilder;
import cn.hutool.core.util.IdUtil;
import com.example.jedis.common.JedisTemplate;
import com.example.jedis.configuration.RedisConfiguration;
import com.example.jedis.model.UserDto;
import com.example.jedis.repository.UserRepository;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;

import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.IntStream;

@SpringBootTest
//@ContextConfiguration(classes = RedisConfiguration.class)
//@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS)
@Slf4j
class SpringbootJedisApplicationTests {

  
    @Autowired
    JedisUtil jedisUtil;

 
    @Test
    void testCrud() {
        IntStream.range(0,100000).forEach(e->{
            final UserDto userDto = UserDto.builder()
                    .id(IdUtil.getSnowflake().nextId())
                    .name("用户1")
                    .gender(UserDto.Gender.MALE)
                    .build();
            userRepository.save(userDto);
        });
    }

 


    @Test
    void testKeys() {
        TimeInterval timeInterval = DateUtil.timer();
        Set<String> setData = jedisUitil.keys("user:*");
        System.out.println("keys use:"+timeInterval.intervalRestart()+"ms");
        Set<String> setDataScan = jedisUitil.scan("user:*");
        System.out.println("scan use:"+timeInterval.intervalRestart()+"ms");
    }
}

使用了3千多额数据，测试keys *和scan其实查询效率差别不大的，scan命令效率和分多少数量一批次也有关系

搞到一万的数据量

经过测试，scan查询效率并不一定是比keys * 快多少的，跟这个数据量和count批次有关系，需要自己调试，所以对于线上的业务场景，如果key数量很多的，可以使用集合来替换keys *

相关推荐
SelectDB7 分钟前
Apache Doris 2025 Roadmap：构建 GenAI 时代实时高效统一的数据底座
大数据·数据库·aigc
用户62799471826220 分钟前
GBase 8a南大通用数据库节点替换
数据库
Мартин.30 分钟前
[CISSP] [5] 保护资产安全
数据库·安全·oracle
熠速30 分钟前
ITTIA DB Platform——实时嵌入式数据管理软件产品家族
数据库·嵌入式实时数据库
热爱编程的小曾1 小时前
sqli-labs靶场 less 8
前端·数据库·less
THRUSTER111111 小时前
MySQL-- 函数（单行函数）：数值函数， 字符串函数
数据库·mysql·函数·navicat·单行函数
橙序研工坊1 小时前
MySQL的进阶语法7（索引-B+Tree 、Hash、聚集索引 、二级索引（回表查询）、索引的使用及设计原则
数据库·sql·mysql
Bruce-li__1 小时前
深入理解Python asyncio：从入门到实战，掌握异步编程精髓
网络·数据库·python
小光学长2 小时前
基于vue框架的智能服务旅游管理系统54kd3（程序+源码+数据库+调试部署+开发环境）带论文文档1万字以上，文末可获取，系统界面在最后面。
数据库
Bonnie_12152 小时前
07-MySQL-事务的隔离级别以及底层原理
数据库·mysql
热门推荐
01我决定放弃搞 Java 了02DeepSeek各版本说明与优缺点分析03如何在WPS和Word/Excel中直接使用DeepSeek功能04RAG 实践- Ollama+RagFlow 部署本地知识库05苍穹外卖面试总结06从零安装 LLaMA-Factory 微调 Qwen 大模型成功及所有的坑07本地化部署AI知识库：基于Ollama+DeepSeek+AnythingLLM保姆级教程08DeepSeek R1本地化部署 Ollama + Chatbox 打造最强 AI 工具09如何本地部署AI智能体平台，带你手搓一个AI Agent10循环神经网络RNN（含LSTM，GRU）小综述