Golang Gin Redis+Mysql 同步查询更新删除操作（我的小GO笔记）

我的需求是在处理几百上千万数据时避免缓存穿透以及缓存击穿情况出现，并且确保数据库和redis同步，为了方便我查询数据操作加了一些条件精准查询和模糊查询以及全字段模糊查询、分页、排序一些小玩意，redis存储是hash表key值也就是数据ID，name值是数据表名和redis同步的，别问为什么，我懒！！

Go 复制代码

使用示例：
params := utils.QueryParams{
	Name:  "users",
	Limit: 10,
	Order: "id",
	Sort:  1,
	Where: map[string]interface{}{
	    "name": "张",  // 将进行模糊查询
	    "age":  18,   // 将进行精确匹配
	    "*": "李",  // 将进行全字段模糊查询
	},
}
results, err := utils.CustomRedisQuery(db, redisClient, params)
if err != nil {
 // 处理错误
}

result, err := GetRedisById(rdb, "users", 1)

result, err := GetRedisByWhere(rdb, "users", map[string]interface{}{"status": 1, "type": "vip"}, 1)

err = DeleteRedisById(rdb, "users", 1)

err = UpdateRedisById(db, rdb, "users", 1)

完整代码

Go 复制代码

/*
+--------------------------------------------------------------------------------
| If this code works, it was written by Xven. If not, I don't know who wrote it.
+--------------------------------------------------------------------------------
| Statement: An Ordinary Person
+--------------------------------------------------------------------------------
| Author: Xven <QQ:270988107>
+--------------------------------------------------------------------------------
| Copyright (c) 2024 Xven All rights reserved.
+--------------------------------------------------------------------------------
*/
package utils

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/go-redis/redis/v8"
	"gorm.io/gorm"
)

type QueryParams struct {
	Name  string                 // 表名
	Limit int                    // 分页数量
	Order string                 // 排序字段
	Sort  int                    // 排序方式 1:升序 2:降序
	Where map[string]interface{} // 查询条件
}

/**
 * 检查字符串是否包含子串
 * @Author Xven　<270988107@qq.com>
 * @param {string} str
 * @param {string} substr
 * @return {bool}
 */
func containsString(str, substr string) bool {
	return strings.Contains(strings.ToLower(str), strings.ToLower(substr))
}

/**
 * 检查字符串是否包含通配符
 * @Author Xven　<270988107@qq.com>
 * @param {string} str
 * @return {bool}
 */
func hasWildcard(str string) bool {
	return strings.Contains(str, "*")
}

/**
 * 对数据进行排序
 * @Author Xven　<270988107@qq.com>
 * @param {[]map[string]interface{}} data
 * @param {string} orderField
 * @param {int} sortType
 * @return {void}
 */
func sortData(data []map[string]interface{}, orderField string, sortType int) {
	sort.Slice(data, func(i, j int) bool {
		if sortType == 1 { // 升序
			return fmt.Sprint(data[i][orderField]) < fmt.Sprint(data[j][orderField])
		}
		return fmt.Sprint(data[i][orderField]) > fmt.Sprint(data[j][orderField])
	})
}

/**
 * 自定义redis查询
 * @Author Xven　<270988107@qq.com>
 * @param {*gorm.DB} db
 * @param {*redis.Client} rdb
 * @param {QueryParams} params
 * @return {[]map[string]interface{}, error}
 */
func CustomRedisQuery(db *gorm.DB, rdb *redis.Client, params QueryParams) ([]map[string]interface{}, error) {
	ctx := context.Background()
	var result []map[string]interface{}

	// 参数校验,防止缓存穿透
	if params.Name == "" {
		return nil, fmt.Errorf("表名不能为空")
	}

	// 构建 Redis key
	redisKey := params.Name + ":list"

	// 使用分布式锁防止缓存击穿
	lockKey := fmt.Sprintf("lock:%s", redisKey)
	lock := rdb.SetNX(ctx, lockKey, "1", 10*time.Second)
	if !lock.Val() {
		// 等待100ms后重试
		time.Sleep(100 * time.Millisecond)
		return CustomRedisQuery(db, rdb, params)
	}
	defer rdb.Del(ctx, lockKey)

	// 1. 先查询 Redis 缓存
	vals, err := rdb.HGetAll(ctx, redisKey).Result()
	if err == nil && len(vals) > 0 {
		// 将缓存数据解析为结果集
		for _, v := range vals {
			var item map[string]interface{}
			if err := json.Unmarshal([]byte(v), &item); err == nil {
				result = append(result, item)
			}
		}

		// 如果有查询条件,则进行过滤
		if len(params.Where) > 0 {
			result = filterData(result, params.Where)
		}

		// 处理排序
		if params.Order != "" {
			sortData(result, params.Order, params.Sort)
		}

		// 处理分页
		if params.Limit > 0 && len(result) > params.Limit {
			result = result[:params.Limit]
		}

		return result, nil
	}

	// 2. Redis 没有数据,从数据库查询
	var dbResult []map[string]interface{}

	// 使用连接池控制并发
	pool := make(chan struct{}, 10)
	var wg sync.WaitGroup
	var mu sync.Mutex

	// 使用游标分批查询数据库,避免一次性加载过多数据
	err = db.Table(params.Name).FindInBatches(&dbResult, 1000, func(tx *gorm.DB, batch int) error {
		wg.Add(1)
		pool <- struct{}{} // 获取连接

		go func(data []map[string]interface{}) {
			defer func() {
				<-pool // 释放连接
				wg.Done()
			}()

			pipe := rdb.Pipeline()

			// 批量写入Redis
			for _, item := range data {
				// 将每条记录序列化为JSON
				jsonData, err := json.Marshal(item)
				if err != nil {
					continue
				}

				// 使用ID作为field,JSON作为value写入hash
				id := fmt.Sprint(item["id"])
				pipe.HSet(ctx, redisKey, id, string(jsonData))
			}

			// 执行管道命令
			_, err := pipe.Exec(ctx)
			if err != nil {
				// 写入失败时重试写入数据
				for _, item := range data {
					jsonData, _ := json.Marshal(item)
					id := fmt.Sprint(item["id"])
					rdb.HSet(ctx, redisKey, id, string(jsonData))
				}
			}

			mu.Lock()
			result = append(result, data...)
			mu.Unlock()

		}(dbResult)

		return nil
	}).Error

	wg.Wait()

	if err != nil {
		// 设置空值缓存,防止缓存穿透
		rdb.Set(ctx, redisKey+"_empty", "1", 5*time.Minute)
		return nil, err
	}

	// 处理排序
	if params.Order != "" {
		sortData(result, params.Order, params.Sort)
	}

	// 处理分页
	if params.Limit > 0 && len(result) > params.Limit {
		result = result[:params.Limit]
	}

	return result, nil
}

/**
 * 过滤数据
 * @Author Xven　<270988107@qq.com>
 * @param {[]map[string]interface{}} data
 * @param {map[string]interface{}} where
 * @return {[]map[string]interface{}}
 */
func filterData(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {
	var filteredResult []map[string]interface{}

	// 先处理精确匹配条件
	hasExactMatch := false
	for field, value := range where {
		if field != "*" {
			if strValue, ok := value.(string); ok && !hasWildcard(strValue) {
				hasExactMatch = true
				break
			} else if !ok {
				hasExactMatch = true
				break
			}
		}
	}

	if hasExactMatch {
		filteredResult = exactMatch(data, where)
		if len(filteredResult) > 0 {
			filteredResult = fuzzyMatch(filteredResult, where)
		}
	} else {
		filteredResult = fuzzyMatch(data, where)
	}

	return filteredResult
}

/**
 * 精确匹配
 * @Author Xven　<270988107@qq.com>
 * @param {[]map[string]interface{}} data
 * @param {map[string]interface{}} where
 * @return {[]map[string]interface{}}
 */
func exactMatch(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {
	var result []map[string]interface{}
	for _, item := range data {
		matched := true
		for field, value := range where {
			if field == "*" {
				continue
			}
			if strValue, ok := value.(string); ok {
				if !hasWildcard(strValue) {
					if itemValue, exists := item[field]; !exists || itemValue != value {
						matched = false
						break
					}
				}
			} else {
				if itemValue, exists := item[field]; !exists || itemValue != value {
					matched = false
					break
				}
			}
		}
		if matched {
			result = append(result, item)
		}
	}
	return result
}

/**
 * 模糊匹配
 * @Author Xven　<270988107@qq.com>
 * @param {[]map[string]interface{}} data
 * @param {map[string]interface{}} where
 * @return {[]map[string]interface{}}
 */
func fuzzyMatch(data []map[string]interface{}, where map[string]interface{}) []map[string]interface{} {
	var result []map[string]interface{}

	// 处理指定字段的模糊查询
	for _, item := range data {
		matched := true
		for field, value := range where {
			if field == "*" {
				continue
			}
			if strValue, ok := value.(string); ok && hasWildcard(strValue) {
				if itemValue, exists := item[field]; exists {
					if strItemValue, ok := itemValue.(string); ok {
						pattern := strings.ReplaceAll(strValue, "*", "")
						if !strings.Contains(strings.ToLower(strItemValue), strings.ToLower(pattern)) {
							matched = false
							break
						}
					}
				}
			}
		}
		if matched {
			result = append(result, item)
		}
	}

	// 处理全字段模糊查询
	if wildcardValue, exists := where["*"]; exists {
		var globalResult []map[string]interface{}
		searchData := result
		if len(searchData) == 0 {
			searchData = data
		}

		if strValue, ok := wildcardValue.(string); ok {
			for _, item := range searchData {
				matched := false
				for _, fieldValue := range item {
					if strFieldValue, ok := fieldValue.(string); ok {
						if containsString(strFieldValue, strValue) {
							matched = true
							break
						}
					}
				}
				if matched {
					globalResult = append(globalResult, item)
				}
			}
		}
		result = globalResult
	}

	return result
}

/**
 * 根据ID查询单条数据
 * @Author Xven　<270988107@qq.com>
 * @param {*redis.Client} rdb
 * @param {string} name
 * @param {interface{}} id
 * @return {map[string]interface{}, error}
 */
func GetRedisById(rdb *redis.Client, name string, id interface{}) (map[string]interface{}, error) {
	ctx := context.Background()
	redisKey := name + ":list"

	// 从Redis查询
	jsonData, err := rdb.HGet(ctx, redisKey, fmt.Sprint(id)).Result()
	if err == nil {
		// Redis命中,解析JSON数据
		var result map[string]interface{}
		err = json.Unmarshal([]byte(jsonData), &result)
		if err == nil {
			return result, nil
		}
	}

	return nil, err
}

/**
 * 根据条件查询数据
 * @Author Xven　<270988107@qq.com>
 * @param {*redis.Client} rdb
 * @param {string} name
 * @param {map[string]interface{}} where
 * @param {int} is
 * @return {interface{}, error}
 */
func GetRedisByWhere(rdb *redis.Client, name string, where map[string]interface{}, is int) (interface{}, error) {
	ctx := context.Background()
	redisKey := name + ":list"

	// 获取所有数据
	values, err := rdb.HGetAll(ctx, redisKey).Result()
	if err != nil {
		return nil, err
	}

	var results []map[string]interface{}
	// 遍历所有数据进行条件匹配
	for _, jsonStr := range values {
		var item map[string]interface{}
		err := json.Unmarshal([]byte(jsonStr), &item)
		if err != nil {
			continue
		}

		// 检查是否匹配所有条件
		match := true
		for k, v := range where {
			if item[k] != v {
				match = false
				break
			}
		}

		if match {
			results = append(results, item)
			// 如果是单条查询且已找到,则直接返回
			if is == 0 {
				return item, nil
			}
		}
	}

	if is == 0 {
		return nil, nil
	}
	return results, nil
}

/**
 * 删除指定ID的数据
 * @Author Xven　<270988107@qq.com>
 * @param {*redis.Client} rdb
 * @param {string} name
 * @param {interface{}} id
 * @return {error}
 */
func DeleteRedisById(rdb *redis.Client, name string, id interface{}) error {
	ctx := context.Background()
	redisKey := name + ":list"
	maxRetries := 3

	for i := 0; i < maxRetries; i++ {
		err := rdb.HDel(ctx, redisKey, fmt.Sprint(id)).Err()
		if err == nil {
			return nil
		}
		// 重试前等待短暂时间
		time.Sleep(time.Millisecond * 100)
	}

	return fmt.Errorf("failed to delete after %d retries", maxRetries)
}

/**
 * 更新指定ID的数据
 * @Author Xven　<270988107@qq.com>
 * @param {*gorm.DB} db
 * @param {*redis.Client} rdb
 * @param {string} name
 * @param {interface{}} id
 * @return {error}
 */
func UpdateRedisById(db *gorm.DB, rdb *redis.Client, name string, id interface{}) error {
	ctx := context.Background()
	redisKey := name + ":list"
	maxRetries := 3

	// 从数据库查询数据
	var result map[string]interface{}
	err := db.Table(name).Where("id = ?", id).Take(&result).Error
	if err != nil {
		return err
	}

	// 序列化数据
	jsonData, err := json.Marshal(result)
	if err != nil {
		return err
	}

	// 重试更新Redis
	for i := 0; i < maxRetries; i++ {
		err = rdb.HSet(ctx, redisKey, fmt.Sprint(id), string(jsonData)).Err()
		if err == nil {
			return nil
		}
		time.Sleep(time.Millisecond * 100)
	}

	return fmt.Errorf("failed to update after %d retries", maxRetries)
}