Linux下Docker Compose里运行PostgreSQL数据库故障诊断Shell脚本

添加执行权限：

bash 复制代码

chmod +x docker_compose_pg_vector_diagnose.sh

自动探测compose内pg容器（推荐）

bash 复制代码

sudo ./docker_compose_pg_vector_diagnose.sh

手动指定容器名+宿主机数据挂载目录

bash 复制代码

sudo ./docker_compose_pg_vector_diagnose.sh postgres_db /data/pg_volume

bash 复制代码

#!/usr/bin/env bash
#===============================================================================
# 脚本名称: docker_compose_pg_vector_diagnose.sh
# 描述: Docker Compose 部署PostgreSQL(含pgvector向量扩展)系统化故障诊断脚本
# 哲学根基:
#   - 本体论: 故障实体建模(容器OOM/进程崩溃/磁盘满/向量索引损坏/权限异常)
#   - 奥卡姆剃刀: 优先校验最高频故障，减少无效采集
#   - 辩证法: 区分紧急恢复 vs 故障现场留存两种处置思路
#   - 实用主义: 自动缓存容器ID/名称/挂载路径，一键复制复用
#   - 整体论: 关联宿主机、容器cgroup、上游流量、存储卷全局排查
#   - 无为: compose配置、内核参数、监控告警长效自愈方案
# 运行要求: root/sudo权限，服务器安装docker、docker-compose、jq
# 传参用法:
#  1. 自动探测compose内pg容器: ./docker_compose_pg_vector_diagnose.sh
#  2. 手动指定容器名+宿主机挂载目录: ./docker_compose_pg_vector_diagnose.sh postgres_db /data/pg_vol
# 特性: 分段可独立执行、缓存关键参数、向量索引专项检测、仅取证不修改数据
#===============================================================================
set -uo pipefail

# =====================【独立可执行块1：全局颜色、缓存文件、工具函数、入参解析】=====================
# 颜色定义 高亮可复制参数
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
WHITE='\033[1;37m'
NC='\033[0m'

# 全局取证缓存目录、参数缓存文件
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
DIAG_DIR="/tmp/pg_docker_diag_${TIMESTAMP}"
CACHE_FILE="${DIAG_DIR}/pg_docker_param_cache.txt"
RECORD_LOG="${DIAG_DIR}/full_diagnose_record.log"
mkdir -p "$DIAG_DIR"
> "$CACHE_FILE"
> "$RECORD_LOG"

# 全局缓存变量（全程复用，执行后可复制）
CONTAINER_NAME=""
CONTAINER_ID=""
PGDATA_HOST=""
PGDATA_CONTAINER="/var/lib/postgresql/data"
CONTAINER_STATE=""
EXIT_CODE=""
OOM_KILLED=""
VECTOR_EXT_ERROR=false

# 基础打印工具
echo_title() {
    echo -e "\n${BLUE}========== $1 ==========${NC}"
    echo -e "\n========== $1 ==========" >> "$RECORD_LOG"
}
echo_ok() {
    echo -e "${GREEN}[✓]${NC} $1"
    echo "[✓] $1" >> "$RECORD_LOG"
}
echo_warn() {
    echo -e "${YELLOW}[⚠]${NC} $1"
    echo "[⚠] $1" >> "$RECORD_LOG"
}
echo_err() {
    echo -e "${RED}[✗]${NC} $1"
    echo "[✗] $1" >> "$RECORD_LOG"
}
run_check() {
    local desc="$1"; shift
    echo -e "${BLUE}>>>${NC} $desc"
    echo "[执行检查] $desc" >> "$RECORD_LOG"
    "$@" 2>&1 | sed 's/^/    /' | tee -a "$RECORD_LOG"
    if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
        echo_warn "上一条命令执行异常，请手工核查"
    fi
}
# 打印红色高亮可复制参数，存入缓存文件
print_copy_param() {
    local label="$1"
    local value="$2"
    echo -e "${WHITE}【可复制参数 | $label】${RED}$value${NC}"
    echo "$label=$value" >> "$CACHE_FILE"
}
# 生成预填充变量的一键执行命令
quick_run_cmd() {
    echo -e "${YELLOW}🔧 一键复制执行：$1${NC}"
    echo "快捷命令：$1" >> "$CACHE_FILE"
}

# 入参读取
CONTAINER_NAME="${1:-}"
PGDATA_HOST="${2:-}"

# 无容器名时自动从docker-compose.yml探测pg/db服务容器
if [[ -z "$CONTAINER_NAME" ]] && [[ -f docker-compose.yml ]]; then
    echo_title "自动探测docker-compose PostgreSQL容器"
    TMP_CONTAINER_ID=$(docker-compose ps -q db 2>/dev/null || docker-compose ps -q postgres 2>/dev/null || echo "")
    if [[ -n "$TMP_CONTAINER_ID" ]]; then
        CONTAINER_ID="$TMP_CONTAINER_ID"
        CONTAINER_NAME=$(docker inspect --format '{{.Name}}' "$CONTAINER_ID" | sed 's/^\///')
        print_copy_param "自动探测容器ID" "$CONTAINER_ID"
        print_copy_param "自动探测容器名称" "$CONTAINER_NAME"
    else
        echo_err "compose文件未探测到db/postgres服务，请手动传入容器名运行脚本"
        exit 1
    fi
elif [[ -n "$CONTAINER_NAME" ]]; then
    # 手动传入容器名，获取对应容器ID
    CONTAINER_ID=$(docker ps -a --filter "name=^/${CONTAINER_NAME}$" --format '{{.Names}} {{.ID}}' | awk '{print $2}')
    print_copy_param "手动传入容器名称" "$CONTAINER_NAME"
    print_copy_param "对应容器ID" "$CONTAINER_ID"
fi

# 基础环境前置校验
echo_title "0. 诊断环境框定 (本体论: 明确故障实体边界)"
if ! command -v docker &> /dev/null; then
    echo_err "Docker 未安装或不在 PATH 中，无法继续诊断"
    exit 1
fi
if ! command -v docker-compose &> /dev/null; then
    echo_warn "未检测docker-compose，仅支持docker原生命令排查，无法读取compose配置"
fi
if [[ -z "$CONTAINER_NAME" || -z "$CONTAINER_ID" ]]; then
    echo_err "容器识别失败，请确认容器名称正确，重新传入参数执行脚本"
    exit 1
fi
# 校验容器真实存在
if ! docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER_NAME"; then
    echo_err "容器 $CONTAINER_NAME 在本机不存在，终止诊断"
    exit 1
fi
echo_ok "目标诊断容器: $CONTAINER_NAME | 容器ID: $CONTAINER_ID"
print_copy_param "PG宿主机数据挂载路径PGDATA_HOST" "$PGDATA_HOST"
print_copy_param "容器内PG数据目录PGDATA_CONTAINER" "$PGDATA_CONTAINER"
# ==========================================================================================

# =====================【独立可执行块2：第一层：容器本体状态判定（退出码/OOM/健康检查）】=====================
echo_title "1. 容器状态 - 本体论：识别故障实体 (退出码决定诊断分支)"
# 读取容器核心状态指标
CONTAINER_STATE=$(docker inspect --format '{{.State.Status}}' "$CONTAINER_NAME")
EXIT_CODE=$(docker inspect --format '{{.State.ExitCode}}' "$CONTAINER_NAME" 2>/dev/null || echo "N/A")
OOM_KILLED=$(docker inspect --format '{{.State.OOMKilled}}' "$CONTAINER_NAME")

echo "   容器运行状态: $CONTAINER_STATE"
echo "   进程退出码: $EXIT_CODE"
echo "   cgroup OOM杀死标记: $OOM_KILLED"
print_copy_param "容器整体运行状态" "$CONTAINER_STATE"
print_copy_param "进程退出码EXIT_CODE" "$EXIT_CODE"
print_copy_param "OOM内核杀死标记OOM_KILLED" "$OOM_KILLED"

# 本体故障实体分类（奥卡姆剃刀优先高频故障）
case "$EXIT_CODE" in
    137)
        echo_err "故障实体: [资源耗尽事件] 进程收到SIGKILL强制终止"
        if [[ "$OOM_KILLED" == "true" ]]; then
            echo_err "根因锁定：容器cgroup内存限制触发OOM Kill（向量高维查询极易触发）"
        else
            echo_err "可疑根因：宿主机全局OOM Killer / 外部手动执行kill -9"
        fi
        echo "诊断优先级：先执行宿主机内存、容器内存限制检查"
        ;;
    1|2)
        echo_err "故障实体: [进程崩溃事件] PostgreSQL程序内部异常退出"
        echo "诊断优先级：优先读取容器日志、pgvector向量索引报错日志"
        ;;
    0)
        echo_warn "故障实体: [正常退出事件] 人为/定时任务执行停止容器，核查运维操作记录"
        ;;
    143)
        echo_warn "故障实体: [优雅终止事件] 收到SIGTERM信号，健康检查失败重启/手动down容器"
        ;;
    *)
        echo_warn "非常规退出码，需结合完整日志综合定位根因"
        ;;
esac

# 容器健康检查历史记录
echo -e "\n${BLUE}>>>${NC} 容器健康检查历史 (最近5次失败记录)"
docker inspect --format '{{range .State.Health.Log}}    {{.Start}} | ExitCode={{.ExitCode}} | {{.Output}}{{println}}{{end}}' "$CONTAINER_NAME" 2>/dev/null || echo "    该容器未配置Docker健康检查"

# 快捷命令：查看容器完整基础信息
quick_run_cmd "docker inspect $CONTAINER_NAME | less"
# ==========================================================================================

# =====================【独立可执行块3：第二层：日志全量采集（普通PG+pgvector向量专属错误匹配）】=====================
echo_title "2. 日志 - 核心故障证据（奥卡姆剃刀：筛选FATAL/PANIC/向量索引报错）"
# 定义错误匹配规则（扩展pgvector向量库特有故障关键词）
CRITICAL_LOG_PATTERN='FATAL|PANIC|ERROR|could not|permission denied|no space|cannot allocate memory|vector index|hnsw|ivfflat|pgvector|high dim memory exhausted|index corrupt'

# 2.1 容器标准输出日志（最近100行，过滤致命错误）
echo -e "${BLUE}>>>${NC} 容器标准输出日志（筛选崩溃/向量索引错误）"
docker logs --tail 100 "$CONTAINER_NAME" 2>&1 | grep -iE "$CRITICAL_LOG_PATTERN" | tail -20 | sed 's/^/    /' | tee -a "$RECORD_LOG"

# 标记是否存在向量索引报错
if docker logs "$CONTAINER_NAME" 2>&1 | grep -qiE "vector index|hnsw|ivfflat|index corrupt"; then
    VECTOR_EXT_ERROR=true
    echo_err "检测到pgvector向量索引损坏/查询内存溢出报错"
fi
print_copy_param "向量扩展故障标记VECTOR_EXT_ERROR" "$VECTOR_EXT_ERROR"

# 2.2 容器内pg数据库日志文件读取（区分运行/停止两种容器状态）
echo -e "\n${BLUE}>>>${NC} 读取PostgreSQL本地日志文件（容器内/log目录）"
if [[ "$CONTAINER_STATE" == "running" ]]; then
    # 容器正常运行，exec进入读取日志
    docker exec "$CONTAINER_NAME" sh -c "tail -50 $PGDATA_CONTAINER/log/postgresql-*.log 2>/dev/null || tail -50 /var/log/postgresql/*.log 2>/dev/null || echo '容器内未找到pg日志'" 2>&1 \
    | grep -iE "$CRITICAL_LOG_PATTERN" | tail -20 | sed 's/^/    /' | tee -a "$RECORD_LOG"
else
    # 容器已停止，从宿主机挂载卷读取日志
    if [[ -n "$PGDATA_HOST" && -d "$PGDATA_HOST" ]]; then
        echo "    容器已离线，从宿主机挂载目录 $PGDATA_HOST 检索日志文件"
        find "$PGDATA_HOST" -name "*.log" -exec tail -50 {} \; 2>/dev/null | grep -iE "$CRITICAL_LOG_PATTERN" | tail -20 | sed 's/^/    /' | tee -a "$RECORD_LOG"
    else
        echo_warn "容器停止且未传入宿主机PG挂载路径，无法读取本地日志文件"
    fi
fi

# 手工完整日志排查命令
echo -e "\n${YELLOW}[手工深度日志排查]${NC} 完整日志无过滤查看："
quick_run_cmd "docker logs $CONTAINER_NAME --tail 800 | less"
# ==========================================================================================

# =====================【独立可执行块4：第三层：宿主机全局资源排查（定向OOM/磁盘/CPU）】=====================
echo_title "3. 宿主机全局资源 - 外部限制根因排查"
# 若标记OOM杀死，优先检查内存相关日志
if [[ "$EXIT_CODE" == "137" || "$OOM_KILLED" == "true" || "$VECTOR_EXT_ERROR" == "true" ]]; then
    echo_err "优先排查内存耗尽问题（向量高维查询极易占用大量内存）"
    run_check "宿主机整体内存使用概览" free -h
    echo -e "\n${BLUE}>>>${NC} 内核dmesg OOM杀死进程记录"
    dmesg -T 2>/dev/null | grep -i "killed process.*postgres\|out of memory" | tail -10 | sed 's/^/    /' | tee -a "$RECORD_LOG"
    run_check "系统日志OOM记录检索" grep -i "out of memory" /var/log/syslog 2>/dev/null | tail -10 || echo "    syslog无OOM记录"
fi

# 磁盘空间&inode检测（数据库崩溃最高频诱因）
echo -e "\n${BLUE}>>>${NC} 磁盘分区空间与inode占用检测"
if [[ -n "$PGDATA_HOST" ]]; then
    run_check "PG数据卷挂载分区磁盘空间" df -h "$PGDATA_HOST"
    run_check "PG数据卷inode使用量" df -i "$PGDATA_HOST"
    echo -e "\n${YELLOW}[手工磁盘清理排查]${NC} 定位大文件目录："
    quick_run_cmd "du -h --max-depth=2 $PGDATA_HOST | sort -rh | head -15"
else
    run_check "系统根分区磁盘空间" df -h /
    run_check "系统根分区inode" df -i /
    echo_warn "未传入PG宿主机挂载目录，无法精准检测数据卷分区"
fi

# CPU负载检测
run_check "宿主机整体负载" uptime
run_check "CPU占用TOP进程" ps aux --sort=-%cpu | head -6
# ==========================================================================================

# =====================【独立可执行块5：第四层：容器cgroup资源限制校验（内存/CPU配额）】=====================
echo_title "4. 容器cgroup资源限制 - 配置层面内存冲突定位"
echo -e "${BLUE}>>>${NC} 容器内存、CPU配额配置"
docker inspect --format '内存硬限制: {{.HostConfig.Memory}} bytes
内存软预留: {{.HostConfig.MemoryReservation}} bytes
CPU核心限制: {{.HostConfig.NanoCpus}}
CPU配额: {{.HostConfig.CpuQuota}}' "$CONTAINER_NAME" | sed 's/^/    /' | tee -a "$RECORD_LOG"

# pg内存参数与容器限制冲突排查（向量库重点）
echo -e "\n${YELLOW}[手工向量内存参数校验]${NC} 进入容器查看postgresql内存配置："
quick_run_cmd "docker exec -it $CONTAINER_NAME psql -U postgres -c 'SHOW shared_buffers; SHOW work_mem; SHOW max_parallel_workers_per_gather;'"
echo "内存冲突判定逻辑：shared_buffers + (work_mem * max_connections) > 容器内存限制 → 必触发OOM"
echo "向量库优化建议：高维检索场景work_mem调至64~128MB，限制并行向量查询数量"
# ==========================================================================================

# =====================【独立可执行块6：第五层：数据卷文件完整性/权限/锁文件检测】=====================
echo_title "5. 数据卷文件系统完整性 - 数据层故障排查"
# 5.1 挂载目录权限校验
if [[ -n "$PGDATA_HOST" ]]; then
    echo -e "${BLUE}>>>${NC} 宿主机PG数据卷目录权限、属主"
    ls -la "$PGDATA_HOST" | head -5 | sed 's/^/    /' | tee -a "$RECORD_LOG"
    echo "权限标准：目录属主UID=999(postgres)，目录权限700"
    echo -e "\n${YELLOW}[手工校验容器内postgres用户UID]${NC}"
    quick_run_cmd "docker exec $CONTAINER_NAME id postgres"
fi

# 5.2 postmaster.pid残留锁文件检测（容器重启失败常见原因）
echo -e "\n${BLUE}>>>${NC} 检测残留postmaster.pid锁文件"
if [[ -n "$PGDATA_HOST" && -f "$PGDATA_HOST/postmaster.pid" ]]; then
    echo_warn "检测到残留postmaster.pid，容器无法正常启动，确认无pg进程后手动删除"
    echo "锁文件内容: $(cat $PGDATA_HOST/postmaster.pid)"
elif [[ "$CONTAINER_STATE" == "running" ]]; then
    docker exec "$CONTAINER_NAME" sh -c "test -f $PGDATA_CONTAINER/postmaster.pid && echo '锁文件存在' || echo '无残留锁文件'" | sed 's/^/    /'
fi

# 5.3 磁盘IO硬件错误检索
run_check "内核日志磁盘IO/文件系统故障检索" dmesg -T 2>/dev/null | grep -iE 'buffer i/o error|ext4|xfs error|remount ro' | tail -10 || echo "    无磁盘硬件报错"

# 5.4 pgvector向量索引专项健康检测（容器运行状态下执行）
if [[ "$CONTAINER_STATE" == "running" ]]; then
    echo -e "\n${BLUE}>>>${NC} pgvector向量索引清单校验"
    quick_run_cmd "docker exec -it $CONTAINER_NAME psql -U postgres -c 'SELECT relname,amname FROM pg_class c JOIN pg_am a ON c.relam=c.relam WHERE a.amname IN (''hnsw'',''ivfflat'');'"
fi
# ==========================================================================================

# =====================【独立可执行块7：第六层：整体论上下游全局关联排查】=====================
echo_title "6. 整体论全局视图 - 跳出容器定位涟漪式故障"
echo "数据库中断通常不是单点问题，需要联动上下游核查："
echo "1. 上游应用：查看应用日志是否存在向量批量查询风暴、短连接打爆连接池"
echo "2. 存储层：NFS/云盘是否延迟飙升、挂载断开、快照备份抢占IO"
echo "3. 监控大盘：故障时间点全服务CPU、内存、磁盘吞吐、网络流量突变"
echo "4. 全链路追踪：Jaeger/Zipkin检索向量检索接口调用量突增记录"

# 容器网络连接数快照（运行状态容器）
if [[ "$CONTAINER_STATE" == "running" ]]; then
    echo -e "\n${BLUE}>>>${NC} 容器当前TCP连接数（排查连接风暴）"
    docker exec "$CONTAINER_NAME" netstat -ant 2>/dev/null | grep -c ESTABLISHED | xargs -I{} echo "    活跃TCP连接数: {}" || echo "    容器未安装netstat工具"
fi
# ==========================================================================================

# =====================【独立可执行块8：第七层：辩证分层恢复方案（普通PG/向量库分开）】=====================
echo_title "7. 分层故障恢复方案（辩证法：停机恢复/在线轻量修复区分）"
echo_ok "===== 一、通用容器PG故障应急恢复 ====="
if [[ "$OOM_KILLED" == "true" ]]; then
    echo_err "OOM内存超限恢复方案："
    echo "1. docker-compose.yml 提升mem_limit内存配额"
    echo "2. 修改postgresql.conf降低work_mem/max_connections"
    echo "3. systemd层面宿主机调大vm.overcommit_memory=1"
    quick_run_cmd "docker-compose down && docker-compose up -d"
fi
if [[ "$EXIT_CODE" == "137" ]]; then
    echo_err "进程被SIGKILL杀死：临时扩容容器内存，限制业务并发向量查询"
fi
if [[ -f "$PGDATA_HOST/postmaster.pid" ]]; then
    echo_warn "残留锁文件处理：确认无pg进程后 rm -f $PGDATA_HOST/postmaster.pid"
fi

echo -e "\n${GREEN}===== 二、pgvector向量索引专属故障恢复 ====="
if [[ "$VECTOR_EXT_ERROR" == "true" ]]; then
    echo_err "向量索引损坏/高维查询OOM修复方案："
    echo "1. 在线低峰修复：REINDEX INDEX CONCURRENTLY 向量索引名;"
    echo "2. 停机完整重建索引：停止容器后执行REINDEX INDEX 索引名;"
    echo "3. 高维查询OOM：降低单次批量向量检索数量，调大work_mem"
    quick_run_cmd "docker exec -it $CONTAINER_NAME psql -U postgres -c 'ALTER SYSTEM SET work_mem = ''128MB''; SELECT pg_reload_conf();'"
fi
# ==========================================================================================

# =====================【独立可执行块9：第八层：无为长效自愈加固方案】=====================
echo_title "8. 无为长效预防 - 从compose配置根除同类故障"
cat << SELFHELP
1. docker-compose.yml资源隔离配置（必加）
    deploy:
      resources:
        limits:
          cpus: '2'
          memory: 4G
        reservations:
          memory: 2G
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 10s
      timeout: 5s
      retries: 3
    restart: unless-stopped
2. pgvector向量库参数优化
    shared_buffers = 1G
    work_mem = 128MB
    max_parallel_workers_per_gather = 2
    max_slot_wal_keep_size = 16GB
3. 可观测性配置
    监控告警：cgroup内存使用率>80%、日志匹配vector index corrupt、容器OOMKill实时推送告警
4. 定时自愈维护
    业务低峰定时执行向量索引REINDEX、清理过期WAL归档文件
5. 混沌演练：预发环境定期注入内存耗尽、磁盘满故障验证自愈能力
SELFHELP
# ==========================================================================================

# =====================【独立可执行块10：收尾：故障现场打包+参数查看指引】=====================
echo_title "诊断收尾：故障现场全量归档打包"
# 打包全部诊断日志、参数缓存
tar -czf "${DIAG_DIR}.tar.gz" -C /tmp "pg_docker_diag_${TIMESTAMP}" 2>/dev/null
echo_ok "✅ PostgreSQL(docker compose+pgvector)完整取证压缩包生成完成"
print_copy_param "故障取证压缩包完整路径" "${DIAG_DIR}.tar.gz"

echo -e "\n${WHITE}【查看全部缓存参数（容器ID/名称/挂载路径）执行命令】${RED}cat ${CACHE_FILE}${NC}"
quick_run_cmd "cat ${CACHE_FILE}"

echo -e "\n${GREEN}【诊断全部执行完成】${NC}"
echo "根因排查优先级：容器OOM Kill > 向量索引损坏 > 磁盘空间耗尽 > 权限异常 > 共享内存分配失败 > 磁盘IO硬件故障"

# =====================【内置全套手工深度排查命令，可单独复制运行】=====================
: <<'MANUAL_GUIDE'
===============================================================================
# 独立手工深度排查命令块（复制直接运行，自动填充缓存变量）
===============================================================================
# 1. 查看compose完整pg服务配置
docker-compose config | grep -A20 postgres

# 2. 进入pg容器交互式终端
docker exec -it $CONTAINER_NAME bash

# 3. 导出所有向量表、索引结构
docker exec $CONTAINER_NAME pg_dump -U postgres -s -f /tmp/vector_schema.sql
docker cp $CONTAINER_NAME:/tmp/vector_schema.sql ${DIAG_DIR}/

# 4. 抓取pg进程崩溃堆栈（运行中容器）
docker exec $CONTAINER_NAME gdb -p $(pidof postgres) --batch -ex "bt" > ${DIAG_DIR}/pg_stack.log

# 5. 重启pg容器
docker-compose restart $CONTAINER_NAME

# 6. 完全重建容器（数据卷保留）
docker-compose down && docker-compose up -d
===============================================================================
MANUAL_GUIDE