KES 故障诊断与应急响应：问题排查、根因分析与应急预案
前言
跟你说个事儿,我干数据库这行十来年了,故障诊断这块儿,真的是谁用谁知道有多头疼。数据库一出问题,那可真是要命,搞不好半夜三更就得爬起来处理。快速定位问题、分析根因、恢复服务,这些都是DBA必须具备的核心能力。
这篇文章呢,我就不跟大家扯那些虚的了,直接上干货。我把自己这些年做故障诊断的经验都掏出来,包括常见问题怎么排查、根因怎么分析、应急响应流程怎么搞,还有一些故障预防的策略。全文都是实际操作,结合了我处理过的真实案例。如果你负责数据库运维,或者需要提升故障处理能力,相信这篇内容对你会有帮助。
一、常见问题诊断
快速定位常见故障。这块儿我踩过不少坑,也总结了一些经验。
连接问题
sql
-- 检查连接数
SELECT count(*) AS total_connections
FROM sys_stat_activity;
-- 检查连接状态
SELECT
state,
count(*) AS count
FROM sys_stat_activity
GROUP BY state
ORDER BY count DESC;
-- 检查连接来源
SELECT
client_addr,
application_name,
count(*) AS count
FROM sys_stat_activity
GROUP BY client_addr, application_name
ORDER BY count DESC;
-- 检查连接失败原因
-- 查看日志
-- grep "connection failed" /data/kingbase/data/sys_log/kingbase-*.log性能问题
sql
-- 查看慢查询
SELECT
pid,
usename,
query,
now() - query_start AS duration
FROM sys_stat_activity
WHERE state = 'active'
AND now() - query_start > INTERVAL '10 seconds'
ORDER BY duration DESC;
-- 查看锁等待
SELECT
blocked.pid AS blocked_pid,
blocked.query AS blocked_query,
blocking.pid AS blocking_pid,
blocking.query AS blocking_query,
now() - blocked.query_start AS wait_duration
FROM sys_stat_activity blocked
JOIN sys_locks l ON blocked.pid = l.pid AND NOT l.granted
JOIN sys_locks granted ON l.locktype = granted.locktype
AND l.database IS NOT DISTINCT FROM granted.database
AND l.relation IS NOT DISTINCT FROM granted.relation
AND granted.granted = true
JOIN sys_stat_activity blocking ON granted.pid = blocking.pid
WHERE blocked.pid != blocking.pid;
-- 查看资源使用
SELECT
pid,
usename,
query,
sys_backend_pid_get_stat_activity(pid) AS activity
FROM sys_stat_activity
WHERE state = 'active';磁盘问题
sql
-- 检查磁盘空间
SELECT
schemaname,
tablename,
pg_size_pretty(pg_total_relation_size(schemaname||'.'||tablename)) AS total_size,
pg_size_pretty(pg_relation_size(schemaname||'.'||tablename)) AS data_size,
pg_size_pretty(pg_indexes_size(schemaname||'.'||tablename::regclass)) AS index_size
FROM sys_tables
WHERE schemaname = 'public'
ORDER BY pg_total_relation_size(schemaname||'.'||tablename) DESC
LIMIT 20;
-- 检查表膨胀
SELECT
schemaname,
tablename,
n_dead_tup,
n_live_tup,
CASE
WHEN n_live_tup = 0 THEN 0
ELSE round(100.0 * n_dead_tup / (n_live_tup + n_dead_tup), 2)
END AS dead_ratio,
last_vacuum,
last_autovacuum
FROM sys_stat_user_tables
WHERE n_dead_tup > 10000
ORDER BY n_dead_tup DESC;
-- 检查索引膨胀
SELECT
schemaname,
tablename,
indexname,
pg_size_pretty(pg_relation_size(indexrelid)) AS index_size,
idx_scan,
CASE
WHEN idx_scan = 0 THEN '未使用'
ELSE '正常'
END AS status
FROM sys_stat_user_indexes
ORDER BY pg_relation_size(indexrelid) DESC;二、根因分析方法
系统化的根因分析方法。
日志分析
bash
#!/bin/bash
# analyze_logs.sh - 日志分析脚本
LOG_DIR="/data/kingbase/data/sys_log"
# 查找错误日志
echo "=== 错误日志 ==="
grep -i "error" $LOG_DIR/kingbase-*.log | tail -20
# 查找死锁
echo "=== 死锁信息 ==="
grep -i "deadlock" $LOG_DIR/kingbase-*.log | tail -10
# 查找慢查询
echo "=== 慢查询 ==="
grep -i "duration:" $LOG_DIR/kingbase-*.log | grep -v "duration: 0" | tail -20
# 查找连接问题
echo "=== 连接问题 ==="
grep -i "connection" $LOG_DIR/kingbase-*.log | grep -i "failed" | tail -10性能快照
sql
-- 创建性能快照函数
CREATE OR REPLACE FUNCTION take_performance_snapshot()
RETURNS void AS $$
BEGIN
-- 记录当前连接
INSERT INTO performance_snapshots (snapshot_time, snapshot_type, snapshot_data)
VALUES (now(), 'connections', (
SELECT json_agg(row_to_json(t))
FROM (
SELECT pid, usename, state, query, now() - query_start AS duration
FROM sys_stat_activity
) t
));
-- 记录锁信息
INSERT INTO performance_snapshots (snapshot_time, snapshot_type, snapshot_data)
VALUES (now(), 'locks', (
SELECT json_agg(row_to_json(t))
FROM (
SELECT * FROM sys_locks
) t
));
-- 记录慢查询
INSERT INTO performance_snapshots (snapshot_time, snapshot_type, snapshot_data)
VALUES (now(), 'slow_queries', (
SELECT json_agg(row_to_json(t))
FROM (
SELECT pid, usename, query, now() - query_start AS duration
FROM sys_stat_activity
WHERE state = 'active' AND now() - query_start > INTERVAL '5 seconds'
) t
));
END;
$$ LANGUAGE plpgsql;
-- 定时采集（每分钟）
-- * * * * * psql -c "SELECT take_performance_snapshot()"系统资源监控
bash
#!/bin/bash
# monitor_system.sh - 系统资源监控
echo "=== CPU使用 ==="
top -bn1 | grep "Cpu(s)" | awk '{print $2 + $4}'
echo "=== 内存使用 ==="
free -h | grep Mem | awk '{print $3 "/" $2}'
echo "=== 磁盘IO ==="
iostat -x 1 1 | grep -E "Device|sd"
echo "=== 网络连接 ==="
netstat -an | grep 54321 | wc -l三、应急响应流程
标准化的应急响应流程。
故障分级
sql
-- 创建故障分级表
CREATE TABLE incident_levels (
level INT PRIMARY KEY,
name VARCHAR(50),
description TEXT,
response_time INTERVAL,
resolution_time INTERVAL
);
INSERT INTO incident_levels VALUES
(1, 'P1-严重', '数据库不可用，业务中断', '5分钟', '30分钟'),
(2, 'P2-重要', '性能严重下降，影响核心业务', '15分钟', '1小时'),
(3, 'P3-一般', '性能下降，不影响核心业务', '30分钟', '4小时'),
(4, 'P4-轻微', '轻微问题，不影响业务', '2小时', '24小时');应急响应脚本
bash
#!/bin/bash
# emergency_response.sh - 应急响应脚本
INCIDENT_LEVEL=$1
INCIDENT_DESC=$2
# 记录故障
echo "$(date) - 故障级别: $INCIDENT_LEVEL, 描述: $INCIDENT_DESC" >> /var/log/incident.log
# 发送告警
send_alert() {
local message=$1
echo "$message" | mail -s "数据库告警" dba@example.com
# 发送短信
curl -X POST "http://sms-api.example.com/send" \
-d "phone=13800138000&message=$message"
}
# 根据故障级别执行不同操作
case $INCIDENT_LEVEL in
P1)
send_alert "P1故障：数据库不可用，立即处理！"
# 检查数据库状态
sys_isready -h localhost -p 54321
if [ $? -ne 0 ]; then
echo "尝试重启数据库..."
systemctl restart kingbase
fi
;;
P2)
send_alert "P2故障：性能严重下降，15分钟内响应"
# 收集性能信息
psql -c "SELECT * FROM sys_stat_activity WHERE state = 'active';" > /tmp/perf_info.txt
psql -c "SELECT * FROM sys_locks;" > /tmp/lock_info.txt
;;
P3)
send_alert "P3故障：性能下降，30分钟内响应"
# 收集慢查询
psql -c "SELECT * FROM sys_stat_activity WHERE now() - query_start > INTERVAL '10 seconds';" > /tmp/slow_queries.txt
;;
P4)
echo "P4故障已记录，将在24小时内处理"
;;
esac故障恢复
bash
#!/bin/bash
# recover.sh - 故障恢复脚本
# 1. 评估故障影响
echo "评估故障影响..."
psql -c "SELECT count(*) FROM sys_stat_activity WHERE state = 'active';"
# 2. 收集故障信息
echo "收集故障信息..."
psql -c "SELECT * FROM sys_stat_activity;" > /tmp/activity.txt
psql -c "SELECT * FROM sys_locks;" > /tmp/locks.txt
psql -c "SELECT * FROM sys_stat_replication;" > /tmp/replication.txt
# 3. 尝试恢复
echo "尝试恢复..."
# 杀掉死锁进程
psql -c "SELECT sys_terminate_backend(pid) FROM sys_stat_activity WHERE state = 'idle in transaction' AND now() - xact_start > INTERVAL '1 hour';"
# 4. 验证恢复
echo "验证恢复..."
psql -c "SELECT 1;"
if [ $? -eq 0 ]; then
echo "数据库恢复成功"
send_alert "数据库已恢复正常"
else
echo "数据库恢复失败，需要进一步处理"
send_alert "数据库恢复失败，需要人工介入"
fi四、故障预防策略
预防胜于治疗。
定期巡检
bash
#!/bin/bash
# daily_check.sh - 日常巡检
echo "=== 日常巡检报告 $(date) ==="
# 1. 检查连接数
echo "1. 连接数检查"
psql -c "SELECT count(*) FROM sys_stat_activity;"
# 2. 检查慢查询
echo "2. 慢查询检查"
psql -c "SELECT query, now() - query_start FROM sys_stat_activity WHERE state = 'active' AND now() - query_start > INTERVAL '10 seconds';"
# 3. 检查磁盘空间
echo "3. 磁盘空间检查"
df -h /data/kingbase
# 4. 检查复制状态
echo "4. 复制状态检查"
psql -c "SELECT * FROM sys_stat_replication;"
# 5. 检查表膨胀
echo "5. 表膨胀检查"
psql -c "SELECT tablename, n_dead_tup, n_live_tup FROM sys_stat_user_tables WHERE n_dead_tup > 10000;"
echo "=== 巡检完成 ==="备份验证
bash
#!/bin/bash
# verify_backup.sh - 备份验证
BACKUP_DIR="/backup/kingbase"
RESTORE_DIR="/tmp/restore_test"
# 1. 恢复备份到测试环境
echo "恢复备份到测试环境..."
rm -rf $RESTORE_DIR/*
tar -xzf $BACKUP_DIR/latest.tar.gz -C $RESTORE_DIR
# 2. 启动测试数据库
echo "启动测试数据库..."
sys_ctl -D $RESTORE_DIR -l /tmp/test.log start
# 3. 验证数据完整性
echo "验证数据完整性..."
psql -h localhost -p 54322 -d test_db -c "SELECT count(*) FROM users;" > /tmp/test_count.txt
# 4. 对比生产数据
echo "对比数据..."
PROD_COUNT=$(psql -h localhost -p 54321 -d prod_db -t -c "SELECT count(*) FROM users;")
TEST_COUNT=$(cat /tmp/test_count.txt)
if [ "$PROD_COUNT" == "$TEST_COUNT" ]; then
echo "备份验证成功"
else
echo "备份验证失败：生产$PROD_COUNT行，测试$TEST_COUNT行"
send_alert "备份验证失败"
fi
# 5. 停止测试数据库
sys_ctl -D $RESTORE_DIR stop五、实战案例解析
场景一：数据库连接耗尽
数据库突然无法连接。
bash
# 问题现象
sys_isready -h localhost -p 54321
# 返回：no response
# 排查步骤
# 1. 检查进程
ps aux | grep kingbase
# 发现大量连接进程
# 2. 查看日志
tail -100 /data/kingbase/data/sys_log/kingbase-*.log
# 发现：too many connections
# 3. 紧急处理
# 修改配置
sed -i 's/max_connections = 200/max_connections = 300/' /data/kingbase/data/kingbase.conf
sys_ctl reload -D /data/kingbase/data
# 4. 重启数据库
systemctl restart kingbase
# 5. 验证恢复
sys_isready -h localhost -p 54321
# 返回：accepting connections场景二：死锁导致业务中断
业务系统频繁报死锁错误。
sql
-- 查看死锁信息
-- 从日志中查找
-- grep "deadlock detected" /data/kingbase/data/sys_log/kingbase-*.log
-- 分析死锁原因
SELECT
blocked.pid AS blocked_pid,
blocked.query AS blocked_query,
blocking.pid AS blocking_pid,
blocking.query AS blocking_query
FROM sys_stat_activity blocked
JOIN sys_locks l ON blocked.pid = l.pid AND NOT l.granted
JOIN sys_locks granted ON l.locktype = granted.locktype
AND l.database IS NOT DISTINCT FROM granted.database
AND l.relation IS NOT DISTINCT FROM granted.relation
AND granted.granted = true
JOIN sys_stat_activity blocking ON granted.pid = blocking.pid;
-- 紧急处理：杀掉死锁进程
SELECT sys_terminate_backend(pid)
FROM sys_stat_activity
WHERE state = 'idle in transaction'
AND now() - xact_start > INTERVAL '5 minutes';
-- 根因修复：统一更新顺序
-- 在应用层代码中，确保按相同顺序访问资源场景三：磁盘空间不足
磁盘空间告警。
bash
# 告警信息
# df -h /data/kingbase
# Filesystem Size Used Avail Use%
# /dev/sda1 100G 95G 5.0G 95%
# 查找大文件
du -sh /data/kingbase/data/* | sort -rh | head -20
# 发现WAL日志占用大量空间
du -sh /data/kingbase/data/sys_wal/*
# 发现7天的WAL日志
# 清理旧WAL日志
# 确保备份完成后
find /data/kingbase/data/sys_wal/ -name "*.backup" -mtime +3 -delete
# 调整WAL保留策略
sed -i 's/wal_keep_segments = 64/wal_keep_segments = 32/' /data/kingbase/data/kingbase.conf
sys_ctl reload -D /data/kingbase/data
# 清理后
df -h /data/kingbase
# Filesystem Size Used Avail Use%
# /dev/sda1 100G 70G 30G 70%总结与展望
故障诊断与应急响应是DBA的核心技能。通过系统化的排查方法、标准化的应急流程和完善的预防策略，可以快速定位问题、恢复服务。
核心原则：
- 建立完善的监控体系，及时发现问题
- 掌握系统化的排查方法，快速定位根因
- 制定标准化的应急流程，规范响应
- 定期演练，提升团队应急能力
- 重视故障预防，减少问题发生
KES提供了丰富的诊断工具和系统视图。在实际应用中，建议建立完善的故障处理机制，持续提升团队的应急响应能力。
期望本篇内容能够帮助你掌握KES故障诊断与应急响应的核心技术，为构建稳定可靠的数据库系统提供技术支撑。