pg数据库一键迁移脚本

pg数据库的一键迁移脚本，废话不多说，直接上脚本：

bash 复制代码

#!/bin/bash

# =============== 配置区域 =============== #
#原数据库信息
SOURCE_HOST="127.0.0.1"
SOURCE_PORT=1111
SOURCE_USER="username"
SOURCE_PASS='password'
#目标数据库信息
TARGET_HOST="127.0.0.2"
TARGET_PORT=2222
TARGET_USER="username"
TARGET_PASS='password'
#数据库库名
DB_NAME="base"
#数据库模式名称
SCHEMA="schema_test"

DUMP_DIR="dump_dir"

# 自动检测系统资源
TOTAL_CORES=$(nproc)
MEM_GB=$(free -g | awk '/Mem:/ {print $2}')

# 根据资源动态配置
THREADS=$(($TOTAL_CORES * 3)) 

# 内存优化（预留2GB给系统）
if [ $MEM_GB -gt 16 ]; then
  MAINT_MEM="$((MEM_GB / 2))GB"
  WORK_MEM="128MB"
  BUFFER_SIZE="128MB"
elif [ $MEM_GB -gt 8 ]; then
  MAINT_MEM="4GB"
  WORK_MEM="64MB"
  BUFFER_SIZE="64MB"
else
  MAINT_MEM="1GB"
  WORK_MEM="32MB"
  BUFFER_SIZE="32MB"
fi

LOG_FILE="db_migration_$(date +%Y%m%d_%H%M%S).log"
# ======================================== #

# 函数：带时间戳的日志记录
log() {
    local log_type=$1
    local message=$2
    local timestamp
    timestamp=$(date +"%Y-%m-%d %H:%M:%S")
    echo "[$timestamp] [$log_type] $message" | tee -a "$LOG_FILE"
}

# 函数：安全执行命令并记录输出
execute_with_log() {
    local command=$1
    local step_name=$2
    local password=$3
    
    log "START" "开始步骤: $step_name"
    
    # 安全记录命令（不显示密码）
    local safe_command=$(echo "$command" | sed "s/$password/*****/g")
    log "CMD" "执行命令: $safe_command"
    
    # 设置密码环境变量
    export PGPASSWORD=$password
    
    # 执行命令并捕获输出和错误
    eval "$command" 2>&1 | while IFS= read -r line; do
        log "PROGRESS" "$line"
    done
    
    local exit_code=${PIPESTATUS[0]}
    if [ $exit_code -ne 0 ]; then
        log "ERROR" "$step_name 失败, 退出码: $exit_code"
        exit $exit_code
    fi
    
    log "COMPLETE" "$step_name 成功完成"
}

# 创建日志文件
touch "$LOG_FILE"
log "INFO" "===== 极速数据库迁移开始 ====="
log "CONFIG" "源数据库: $SOURCE_HOST:$SOURCE_PORT"
log "CONFIG" "目标数据库: $TARGET_HOST:$TARGET_PORT"
log "CONFIG" "数据库: $DB_NAME, 模式: $SCHEMA"
log "CONFIG" "自动线程数: $THREADS"
log "CONFIG" "内存配置: 维护内存=$MAINT_MEM, 工作内存=$WORK_MEM"
log "CONFIG" "缓冲区: $BUFFER_SIZE"
log "CONFIG" "日志文件: $LOG_FILE"

# ================== 导出阶段 ================== #
log "INFO" "---- 阶段1: 并行数据导出 ----"
log "INFO" "优化策略：零压缩 + 大表优先"

export_command="pg_dump -h $SOURCE_HOST -p $SOURCE_PORT -U $SOURCE_USER -Fd \
  -j $THREADS --compress=0 -f $DUMP_DIR -n $SCHEMA $DB_NAME \
  --load-via-partition-root"

execute_with_log "$export_command" "源数据导出" "$SOURCE_PASS"

# 获取导出目录大小
dump_size=$(du -sh "$DUMP_DIR" 2>/dev/null | cut -f1)
log "INFO" "导出数据大小: ${dump_size:-未知}"

# ================== 导入准备 ================== #
log "INFO" "---- 阶段2: 目标数据库极限优化 ----"
log "INFO" "启用写入性能模式 (临时牺牲ACID特性)"

(
export PGPASSWORD=$TARGET_PASS
psql -h $TARGET_HOST -p $TARGET_PORT -U $TARGET_USER -d $DB_NAME <<EOF >> "$LOG_FILE" 2>&1
-- 禁用ACID特性 (导入期间临时关闭)
ALTER SYSTEM SET fsync = off;
ALTER SYSTEM SET full_page_writes = off;
ALTER SYSTEM SET synchronous_commit = off;
ALTER SYSTEM SET wal_level = minimal;

-- 内存优化
ALTER SYSTEM SET max_wal_size = '8GB';
ALTER SYSTEM SET checkpoint_timeout = '2h';
ALTER SYSTEM SET maintenance_work_mem = '$MAINT_MEM';
ALTER SYSTEM SET work_mem = '$WORK_MEM';
ALTER SYSTEM SET shared_buffers = '$BUFFER_SIZE';

-- 并行度优化
ALTER SYSTEM SET max_parallel_maintenance_workers = $THREADS;
ALTER SYSTEM SET max_parallel_workers = $THREADS;
ALTER SYSTEM SET max_parallel_workers_per_gather = $THREADS;

-- 立即生效
SELECT pg_reload_conf();
EOF
)

log "INFO" "目标数据库已进入极限性能模式"

# ================== 导入阶段 ================== #
log "INFO" "---- 阶段3: 分阶段并行恢复 ----"

# 第一阶段：仅恢复表结构 (加速DDL)
log "INFO" "步骤3.1: 恢复表结构 (并行)"
import_structure_command="pg_restore -h $TARGET_HOST -p $TARGET_PORT -U $TARGET_USER -d $DB_NAME \
  --schema=$SCHEMA --clean --if-exists --no-owner --no-acl \
  -j $THREADS --section=pre-data $DUMP_DIR"

execute_with_log "$import_structure_command" "表结构恢复" "$TARGET_PASS"

# 第二阶段：并行导入数据 (禁用所有约束)
log "INFO" "步骤3.2: 并行数据导入 (禁用所有约束)"
import_data_command="pg_restore -h $TARGET_HOST -p $TARGET_PORT -U $TARGET_USER -d $DB_NAME \
  --schema=$SCHEMA --no-owner --no-acl \
  -j $THREADS --section=data --disable-triggers --data-only $DUMP_DIR"

execute_with_log "$import_data_command" "数据导入" "$TARGET_PASS"

# 第三阶段：并行重建索引
log "INFO" "步骤3.3: 并行重建索引"
import_index_command="pg_restore -h $TARGET_HOST -p $TARGET_PORT -U $TARGET_USER -d $DB_NAME \
  --schema=$SCHEMA --no-owner --no-acl \
  -j $THREADS --section=post-data $DUMP_DIR"

execute_with_log "$import_index_command" "索引重建" "$TARGET_PASS"

# =============== 后期优化 ================ #
log "INFO" "---- 阶段4: 后期优化 ----"
log "INFO" "重建统计信息并恢复ACID特性"

(
export PGPASSWORD=$TARGET_PASS
psql -h $TARGET_HOST -p $TARGET_PORT -U $TARGET_USER -d $DB_NAME <<EOF >> "$LOG_FILE" 2>&1
-- 重建统计信息 (使用并行)
SET maintenance_work_mem = '$MAINT_MEM';
ANALYZE VERBOSE;

-- 恢复ACID特性
ALTER SYSTEM SET fsync = on;
ALTER SYSTEM SET full_page_writes = on;
ALTER SYSTEM SET synchronous_commit = on;
ALTER SYSTEM SET wal_level = replica;

-- 恢复默认并行设置
ALTER SYSTEM RESET max_parallel_maintenance_workers;
ALTER SYSTEM RESET max_parallel_workers;
ALTER SYSTEM RESET max_parallel_workers_per_gather;

-- 恢复内存设置
ALTER SYSTEM RESET maintenance_work_mem;
ALTER SYSTEM RESET work_mem;
ALTER SYSTEM RESET shared_buffers;

-- 刷新配置
SELECT pg_reload_conf();

-- 强制检查点确保数据持久化
CHECKPOINT;
EOF
)

# 清理敏感信息
unset SOURCE_PASS
unset TARGET_PASS
unset PGPASSWORD

# =============== 迁移完成 ================== #
log "SUCCESS" "===== 数据库迁移成功完成! ====="
log "PERF" "总耗时: $SECONDS 秒"
if [ -n "$dump_size" ]; then
  dump_bytes=$(du -sb "$DUMP_DIR" 2>/dev/null | awk '{print $1}')
  if [ -n "$dump_bytes" ] && [ "$dump_bytes" -gt 0 ]; then
    rate_kbs=$((dump_bytes / SECONDS / 1024))
    rate_mbs=$((dump_bytes / SECONDS / 1048576))
    log "PERF" "平均速率: $rate_kbs KB/s ($rate_mbs MB/s)"
  else
    log "PERF" "速率计算: 无法确定导出大小"
  fi
else
  log "PERF" "速率计算: 无法确定导出大小"
fi
log "INFO" "详细日志请查看: $LOG_FILE"
log "INFO" "导出目录: $DUMP_DIR (可手动删除以释放空间)"

上述就是脚本的具体信息，改一下目标数据库和原数据的信息就OK。

使用如下命令执行脚本：

bash 复制代码

nohup sh postgre.sh > migration.log 2>&1 &

然后就可以在migration.log中查看整个迁移过程的日志。

完事，收工。