spring boot 八、 sharding-jdbc 分库分表 按月分表

等一场春雨2023-09-25 10:06

在项目resources目录下新建com.jianmu.config.sharding.DateShardingAlgorithm 文件


新增yaml配置 数据源

yaml 复制代码 
spring:
  shardingsphere:
    props:
      sql:
        #是否在日志中打印 SQL
        show: true
        #打印简单风格的 SQL
        simple: true
    datasource:
      names: pingxuanlog
      pingxuanlog:
        type: com.alibaba.druid.pool.DruidDataSource
        driver-class-name: com.mysql.jdbc.Driver
        url: jdbc:mysql://localhost:3306/db_jianmu_pingxuan_log?serverTimezone=UTC&useSSL=false&useUnicode=true&characterEncoding=UTF-8
        username: root
        password: root
        #最大连接池数量
        max-active: 10
        #最小连接池数量
        min-idle: 5
        #初始化时建立物理连接的个数
        initial-size: 5
        #获取连接时最大等待时间，单位毫秒
        max-wait: 3000
        #配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒
        time-between-eviction-runs-millis: 60000
        #配置一个连接在池中最小生存的时间，单位是毫秒
        min-evictable-idle-time-millis: 100000
        #用来检测连接是否有效的sql，要求是一个查询语句
        validation-query: SELECT 1 FROM DUAL
        #建议配置为true，不影响性能，并且保证安全性。申请连接的时候检测，如果空闲时间大于timeBetweenEvictionRunsMillis，执行validationQuery检测连接是否有效。
        test-while-idle: true

新增yaml配置 sharding 分表规则

yaml 复制代码 
spring:
  shardingsphere:
    sharding:
      tables:
        t_act_vt_log:
          #配置数据节点，这里是按月分表，时间范围设置在202201 ~ 210012
          actual-data-nodes: pingxuanlog.t_act_vt_log_$->{202201..203012}
          table-strategy:
            standard:
              #使用标准分片策略，配置分片字段
              sharding-column: add_time
              # 精确匹配规则(自定义类)
              precise-algorithm-class-name: com.jianmu.config.sharding.DateShardingAlgorithm
              # 范围匹配规则(自定义类)
              range-algorithm-class-name: com.jianmu.config.sharding.DateShardingAlgorithm
        t_act_access_log:
          #配置数据节点，这里是按月分表，时间范围设置在202201 ~ 210012
          actual-data-nodes: pingxuanlog.t_act_access_log_$->{202201..203012}
          table-strategy:
            standard:
              #使用标准分片策略，配置分片字段
              sharding-column: add_time
              # 精确匹配规则(自定义类)
              precise-algorithm-class-name: com.jianmu.config.sharding.DateShardingAlgorithm
              # 范围匹配规则(自定义类)
              range-algorithm-class-name: com.jianmu.config.sharding.DateShardingAlgorithm

DataSourceConfiguration

java 复制代码 
package com.jianmu.config.sharding;

import com.baomidou.dynamic.datasource.DynamicRoutingDataSource;
import com.baomidou.dynamic.datasource.provider.AbstractDataSourceProvider;
import com.baomidou.dynamic.datasource.provider.DynamicDataSourceProvider;
import com.baomidou.dynamic.datasource.spring.boot.autoconfigure.DataSourceProperty;
import com.baomidou.dynamic.datasource.spring.boot.autoconfigure.DynamicDataSourceAutoConfiguration;
import com.baomidou.dynamic.datasource.spring.boot.autoconfigure.DynamicDataSourceProperties;
import org.apache.shardingsphere.shardingjdbc.jdbc.core.datasource.ShardingDataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.SpringBootConfiguration;
import org.springframework.boot.autoconfigure.AutoConfigureBefore;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;

import javax.sql.DataSource;
import java.util.Map;

/**
 * 动态数据源配置：
 * <p>
 * 使用{@link com.baomidou.dynamic.datasource.annotation.DS}注解，切换数据源
 *
 * <code>@DS(DataSourceConfiguration.SHARDING_DATA_SOURCE_NAME)</code>
 *
 */
@Configuration
@AutoConfigureBefore({DynamicDataSourceAutoConfiguration.class, SpringBootConfiguration.class})
public class DataSourceConfiguration {
    /**
     * 分表数据源名称
     */
    public static final String SHARDING_DATA_SOURCE_NAME = "sharding";
    /**
     * 动态数据源配置项
     */
    private final DynamicDataSourceProperties dynamicDataSourceProperties;
    private final ShardingDataSource shardingDataSource;

    @Autowired
    public DataSourceConfiguration(DynamicDataSourceProperties dynamicDataSourceProperties, @Lazy ShardingDataSource shardingDataSource) {
        this.dynamicDataSourceProperties = dynamicDataSourceProperties;
        this.shardingDataSource = shardingDataSource;
    }


    /**
     * 将shardingDataSource放到了多数据源（dataSourceMap）中
     *
     */
    @Bean
    public DynamicDataSourceProvider dynamicDataSourceProvider() {
        Map<String, DataSourceProperty> datasourceMap = dynamicDataSourceProperties.getDatasource();
        return new AbstractDataSourceProvider() {
            @Override
            public Map<String, DataSource> loadDataSources() {
                Map<String, DataSource> dataSourceMap = createDataSourceMap(datasourceMap);
                // 将 sharding jdbc 管理的数据源也交给动态数据源管理
                dataSourceMap.put(SHARDING_DATA_SOURCE_NAME, shardingDataSource);
                return dataSourceMap;
            }
        };
    }

    /**
     * 将动态数据源设置为首选的
     * 当spring存在多个数据源时, 自动注入的是首选的对象
     * 设置为主要的数据源之后，就可以支持sharding jdbc原生的配置方式
     */
    @Primary
    @Bean
    public DataSource dataSource(DynamicDataSourceProvider dynamicDataSourceProvider) {
        DynamicRoutingDataSource dataSource = new DynamicRoutingDataSource();
        dataSource.setPrimary(dynamicDataSourceProperties.getPrimary());
        dataSource.setStrict(dynamicDataSourceProperties.getStrict());
        dataSource.setStrategy(dynamicDataSourceProperties.getStrategy());
        dataSource.setProvider(dynamicDataSourceProvider);
        dataSource.setP6spy(dynamicDataSourceProperties.getP6spy());
        dataSource.setSeata(dynamicDataSourceProperties.getSeata());
        return dataSource;
    }
}
java 复制代码 
package com.jianmu.config.sharding;

import org.springframework.beans.factory.ObjectProvider;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.actuate.autoconfigure.jdbc.DataSourceHealthContributorAutoConfiguration;
import org.springframework.boot.actuate.health.AbstractHealthIndicator;
import org.springframework.boot.actuate.jdbc.DataSourceHealthIndicator;
import org.springframework.boot.jdbc.metadata.DataSourcePoolMetadataProvider;
import org.springframework.context.annotation.Configuration;
import org.springframework.util.StringUtils;

import javax.sql.DataSource;
import java.util.Map;

/**
 * @author kong
 */
@Configuration
public class DataSourceHealthConfig extends DataSourceHealthContributorAutoConfiguration {

    @Value("${spring.datasource.dbcp2.validation-query:select 1}")
    private String defaultQuery;


    public DataSourceHealthConfig(Map<String, DataSource> dataSources, ObjectProvider<DataSourcePoolMetadataProvider> metadataProviders) {
        super(dataSources, metadataProviders);
    }

    @Override
    protected AbstractHealthIndicator createIndicator(DataSource source) {
        DataSourceHealthIndicator indicator = (DataSourceHealthIndicator) super.createIndicator(source);
        if (!StringUtils.hasText(indicator.getQuery())) {
            indicator.setQuery(defaultQuery);
        }
        return indicator;
    }
}
java 复制代码 
package com.jianmu.config.sharding;


import com.alibaba.fastjson2.JSON;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import com.jianmu.tools.ApplicationContextTools;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.standard.PreciseShardingValue;
import org.apache.shardingsphere.api.sharding.standard.RangeShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.standard.RangeShardingValue;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collection;
import java.util.List;
import java.util.stream.Collectors;

/**
	sharding  分库分表策略配置：按照年月 如t_act_vt_log_202208
 **/
@Slf4j
public class DateShardingAlgorithm implements PreciseShardingAlgorithm<LocalDateTime>, RangeShardingAlgorithm<LocalDateTime> {
    private final DateTimeFormatter format = DateTimeFormatter.ofPattern("yyyyMM");

    @Override
    public String doSharding(Collection<String> availableTargetNames, PreciseShardingValue<LocalDateTime> psv) {
        final String table = psv.getLogicTableName();
        final String prefix = table + "_";
        if (log.isDebugEnabled()) {
            log.debug("分表更新");
        }
        TableService tableService = (TableService) ApplicationContextTools.getBean(TableService.class);
        //添加日期
        final String logicTable = prefix + psv.getValue().format(this.format);
        boolean exist = tableService.exist(logicTable);
        if (log.isDebugEnabled()) {
            log.debug("更新: {} 存在:{}", logicTable, exist);
        }
        //匹配到了
        if (exist) {
            return logicTable;
        }
        //未匹配到
        if (log.isDebugEnabled()) {
            log.debug("sharding 未匹配到表，需要创建");
        }
        //创建这张表
        tableService.copy(logicTable, table);
        return logicTable;

    }

    /**
     * 范围匹配
     */
    @SneakyThrows
    @Override
    public Collection<String> doSharding(Collection<String> collection, RangeShardingValue<LocalDateTime> rsv) {
        final String table = rsv.getLogicTableName();
        final String prefix = table + "_";
        if (log.isDebugEnabled()) {
            log.debug("分表查询 表:{}", table);
        }

        TableService tableService = (TableService) ApplicationContextTools.getBean(TableService.class);
        List<String> tables = tableService.tables();

        Range<LocalDateTime> range = rsv.getValueRange();
        //计算出时间范围内的所有日期
        final int upper = range.hasUpperBound() ? Integer.parseInt(range.upperEndpoint().toLocalDate().format(this.format)) : 0;
        final int lower = range.hasLowerBound() ? Integer.parseInt(range.lowerEndpoint().toLocalDate().format(this.format)) : 0;
        List<String> validTables = this.validTables(prefix, tables, upper, lower);

        if (log.isDebugEnabled()) {
            log.debug("查询表:{}", JSON.toJSONString(validTables));
        }
        if (CollectionUtils.isEmpty(validTables)) {
            if (log.isDebugEnabled()) {
                log.debug("查询表不存在 改查原始表");
            }
            return Lists.newArrayList(table);
        }
        return validTables;
    }

    private List<String> validTables(String prefix, List<String> tables, int upper, int lower) {
        if (log.isDebugEnabled()) {
            log.debug("上界:{},下界:{}", upper, lower);
        }
        return tables.parallelStream().filter(i -> {
            if (i.contains(prefix)) {
                final String date = i.replace(prefix, "");
                if (date.matches("[0-9]*")) {
                    final int mouth = Integer.parseInt(date);
                    if (upper > 0 && lower > 0) {
                        return mouth <= upper && mouth >= lower;
                    } else {
                        if (upper > 0) {
                            return mouth <= upper;
                        }
                        if (lower > 0) {
                            return mouth >= lower;
                        }
                    }
                }
            }
            return false;
        }).collect(Collectors.toList());
    }

}
java 复制代码 
package com.jianmu.config.sharding;

public class ShardingConstant {
    public static final String ORIGINAL_DATABASE = "db_jianmu_pingxuan_log";
}
java 复制代码 
package com.jianmu.config.sharding;

import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.ExecutionException;

public interface TableService {


    /**
     * 返回所有表
     */
    List<String> tables() throws ExecutionException, SQLException;

    boolean exist(String table);

    /**
     * 精确删除表
     */
    boolean drop(String table);

    /**
     * 旧表复制为新表
     */
    boolean copy(String newTable, String usedTable);
}
java 复制代码 
package com.jianmu.config.sharding;

import com.baomidou.dynamic.datasource.DynamicRoutingDataSource;
import com.baomidou.dynamic.datasource.annotation.DS;
import com.jianmu.constant.SysRedisConstant;
import com.jianmu.mapper.system.TableMapper;
import com.jianmu.tools.JdbcTools;
import com.jianmu.tools.log.LogTools;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.stereotype.Service;

import java.sql.SQLException;
import java.util.List;

@Slf4j
@Service
@DS("sharding")
public class TableServiceImpl implements TableService {
    private final TableMapper tableMapper;
    private final RedisTemplate<String, Object> redis;
    private final ValueOperations<String, Object> redisValue;
    private final DynamicRoutingDataSource dynamicRoutingDataSource;

    @Autowired
    public TableServiceImpl(TableMapper tableMapper, RedisTemplate<String, Object> redis, DynamicRoutingDataSource dynamicRoutingDataSource) {
        this.tableMapper = tableMapper;
        this.redis = redis;
        this.redisValue = redis.opsForValue();
        this.dynamicRoutingDataSource = dynamicRoutingDataSource;
    }

    @Override
    public List<String> tables() throws SQLException {
        List<String> tables = (List<String>) this.redisValue.get(SysRedisConstant.TABLES);
        if (tables == null) {
            List<String> list = JdbcTools.tables(this.dynamicRoutingDataSource.getDataSource(DataSourceConfiguration.SHARDING_DATA_SOURCE_NAME).getConnection(), ShardingConstant.ORIGINAL_DATABASE);
            //每24小时更新一次
            this.redisValue.set(SysRedisConstant.TABLES, list, 86400);
            return list;
        }
        return tables;
    }

    @Override
    public boolean exist(String table) {
        try {
            Integer exist = this.tableMapper.exist(table);
            if (log.isDebugEnabled()) {
                log.debug("表:{} 存在:{}", table, exist);
            }
            return true;
        } catch (Exception e) {
            LogTools.err(e);
            return false;
        }
    }

    @Override
    public boolean drop(String table) {
        boolean flag = this.tableMapper.drop(table) >= 0;
        if (flag) {
            this.delCache();
        }
        return flag;
    }


    @Override
    public boolean copy(String newTable, String usedTable) {
        boolean flag = this.tableMapper.copy(newTable, usedTable) >= 0;
        if (flag) {
            this.delCache();
        }
        return flag;
    }

    private void delCache() {
        this.redis.delete(SysRedisConstant.TABLES);
    }

}

t_act_vt_log 为每次自动生成的基础表 需要在这个表建好索引等

相关推荐
Asthenia04128 小时前
Spring AOP 和 Aware：在Bean实例化后-调用BeanPostProcessor开始工作！在初始化方法执行之前！
后端
Asthenia04129 小时前
什么是消除直接左递归 - 编译原理解析
后端
Asthenia04129 小时前
什么是自上而下分析 - 编译原理剖析
后端
Asthenia04129 小时前
什么是语法分析 - 编译原理基础
后端
Asthenia04129 小时前
理解词法分析与LEX：编译器的守门人
后端
uhakadotcom9 小时前
视频直播与视频点播：基础知识与应用场景
后端·面试·架构
Asthenia041210 小时前
Spring扩展点与工具类获取容器Bean-基于ApplicationContextAware实现非IOC容器中调用IOC的Bean
后端
bobz96510 小时前
ovs patch port 对比 veth pair
后端
Asthenia041211 小时前
Java受检异常与非受检异常分析
后端
热门推荐
01我决定放弃搞 Java 了02DeepSeek各版本说明与优缺点分析03如何在WPS和Word/Excel中直接使用DeepSeek功能04汽车上的各种质量：整备质量、总质量、装载质量、簧上质量、簧下质量05RAG 实践- Ollama+RagFlow 部署本地知识库06从零安装 LLaMA-Factory 微调 Qwen 大模型成功及所有的坑07【AUTOSAR 基础软件】Can模块详解（Can栈之驱动模块）08本地化部署AI知识库：基于Ollama+DeepSeek+AnythingLLM保姆级教程09DeepSeek RAGFlow构建本地知识库系统10Mac cursor设置jdk、Maven版本