服务器操作卡顿问题解决

1、目前使用时发现操作明显有卡顿感 常用的方法,进程角度

# 综合视图，按 CPU 排序
ps aux --sort=-%cpu | head -20

# 按内存排序
ps aux --sort=-%mem | head -20

# 线程级查看（找多线程程序的瓶颈线程）
top -H -p <PID>  # 然后按 P 排序看 CPU，按 M 排序看内存

# 进程树，看父子关系
pstree -p <PID>

hanwang@k8s-master-node:~/work/ai-coding$ ps aux --sort=-%mem | head -20
USER         PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
libvirt+    4186 11.4 28.3 15033916 8392752 ?    Sl   3月26 406:23 /usr/bin/qemu-system-x86_64 -name guest=devstack-vm,debug-threads=on -S -object {"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-1-devstack-vm/master-key.aes"} -machine pc-q35-8.2,usb=off,dump-guest-core=off,memory-backend=pc.ram,hpet=off,acpi=on -accel kvm -cpu host,migratable=on -m size=8388608k -object {"qom-type":"memory-backend-ram","id":"pc.ram","size":8589934592} -overcommit mem-lock=off -smp 4,sockets=4,cores=1,threads=1 -uuid 5e6f16c3-0a69-412e-a303-ac90d506e8f4 -display none -no-user-config -nodefaults -chardev socket,id=charmonitor,fd=32,server=on,wait=off -mon chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew -global kvm-pit.lost_tick_policy=delay -no-shutdown -global ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on -device {"driver":"pcie-root-port","port":8,"chassis":1,"id":"pci.1","bus":"pcie.0","multifunction":true,"addr":"0x1"} -device {"driver":"pcie-root-port","port":9,"chassis":2,"id":"pci.2","bus":"pcie.0","addr":"0x1.0x1"} -device {"driver":"pcie-root-port","port":10,"chassis":3,"id":"pci.3","bus":"pcie.0","addr":"0x1.0x2"} -device {"driver":"pcie-root-port","port":11,"chassis":4,"id":"pci.4","bus":"pcie.0","addr":"0x1.0x3"} -device {"driver":"pcie-root-port","port":12,"chassis":5,"id":"pci.5","bus":"pcie.0","addr":"0x1.0x4"} -device {"driver":"pcie-root-port","port":13,"chassis":6,"id":"pci.6","bus":"pcie.0","addr":"0x1.0x5"} -device {"driver":"pcie-root-port","port":14,"chassis":7,"id":"pci.7","bus":"pcie.0","addr":"0x1.0x6"} -device {"driver":"pcie-root-port","port":15,"chassis":8,"id":"pci.8","bus":"pcie.0","addr":"0x1.0x7"} -device {"driver":"pcie-root-port","port":16,"chassis":9,"id":"pci.9","bus":"pcie.0","multifunction":true,"addr":"0x2"} -device {"driver":"pcie-root-port","port":17,"chassis":10,"id":"pci.10","bus":"pcie.0","addr":"0x2.0x1"} -device {"driver":"pcie-root-port","port":18,"chassis":11,"id":"pci.11","bus":"pcie.0","addr":"0x2.0x2"} -device {"driver":"pcie-root-port","port":19,"chassis":12,"id":"pci.12","bus":"pcie.0","addr":"0x2.0x3"} -device {"driver":"pcie-root-port","port":20,"chassis":13,"id":"pci.13","bus":"pcie.0","addr":"0x2.0x4"} -device {"driver":"pcie-root-port","port":21,"chassis":14,"id":"pci.14","bus":"pcie.0","addr":"0x2.0x5"} -device {"driver":"qemu-xhci","p2":15,"p3":15,"id":"usb","bus":"pci.2","addr":"0x0"} -device {"driver":"virtio-serial-pci","id":"virtio-serial0","bus":"pci.3","addr":"0x0"} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/devstack/base.img","node-name":"libvirt-3-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-3-format","read-only":true,"discard":"unmap","driver":"qcow2","file":"libvirt-3-storage","backing":null} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/devstack-vm-1.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-2-format","read-only":false,"discard":"unmap","driver":"qcow2","file":"libvirt-2-storage","backing":"libvirt-3-format"} -device {"driver":"virtio-blk-pci","bus":"pci.4","addr":"0x0","drive":"libvirt-2-format","id":"virtio-disk0","bootindex":1} -blockdev {"driver":"file","filename":"/var/lib/libvirt/images/devstack/seed.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"} -blockdev {"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"} -device {"driver":"ide-cd","bus":"ide.0","drive":"libvirt-1-format","id":"sata0-0-0"} -netdev {"type":"tap","fd":"30","vhost":true,"vhostfd":"36","id":"hostnet0"} -device {"driver":"virtio-net-pci","netdev":"hostnet0","id":"net0","mac":"52:54:00:f6:f2:55","bus":"pci.1","addr":"0x0"} -chardev pty,id=charserial0 -device {"driver":"isa-serial","chardev":"charserial0","id":"serial0","index":0} -chardev socket,id=charchannel0,fd=31,server=on,wait=off -device {"driver":"virtserialport","bus":"virtio-serial0.0","nr":1,"chardev":"charchannel0","id":"channel0","name":"org.qemu.guest_agent.0"} -audiodev {"id":"audio1","driver":"none"} -global ICH9-LPC.noreboot=off -watchdog-action reset -device {"driver":"virtio-balloon-pci","id":"balloon0","bus":"pci.5","addr":"0x0"} -object {"qom-type":"rng-random","id":"objrng0","filename":"/dev/urandom"} -device {"driver":"virtio-rng-pci","rng":"objrng0","id":"rng0","bus":"pci.6","addr":"0x0"} -sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny -msg timestamp=on
hanwang     8557  0.2 15.3 14077228 4546868 ?    Sl   3月26   7:47 /usr/share/elasticsearch/jdk/bin/java -Des.networkaddress.cache.ttl=60 -Des.networkaddress.cache.negative.ttl=10 -Djava.security.manager=allow -XX:+AlwaysPreTouch -Xss1m -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Djna.nosys=true -XX:-OmitStackTraceInFastThrow -Dio.netty.noUnsafe=true -Dio.netty.noKeySetOptimization=true -Dio.netty.recycler.maxCapacityPerThread=0 -Dlog4j.shutdownHookEnabled=false -Dlog4j2.disable.jmx=true -Dlog4j2.formatMsgNoLookups=true -Djava.locale.providers=SPI,COMPAT --add-opens=java.base/java.io=org.elasticsearch.preallocate -Des.cgroups.hierarchy.override=/ -XX:+UseG1GC -Djava.io.tmpdir=/tmp/elasticsearch-14664739578300384182 --add-modules=jdk.incubator.vector -XX:+HeapDumpOnOutOfMemoryError -XX:+ExitOnOutOfMemoryError -XX:HeapDumpPath=data -XX:ErrorFile=logs/hs_err_pid%p.log -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,level,pid,tags:filecount=32,filesize=64m -Xms3849m -Xmx3849m -XX:MaxDirectMemorySize=2018508800 -XX:G1HeapRegionSize=4m -XX:InitiatingHeapOccupancyPercent=30 -XX:G1ReservePercent=15 -Des.distribution.type=docker --module-path /usr/share/elasticsearch/lib --add-modules=jdk.net --add-modules=ALL-MODULE-PATH -m org.elasticsearch.server/org.elasticsearch.bootstrap.Elasticsearch
hanwang   625102 13.3  2.0 74796540 616464 pts/5 Tl   09:50   0:11 /usr/lib/node_modules/opencode-ai/bin/.opencode
hanwang     7571  0.9  1.9 726924 585612 ?       Ss   3月26  32:17 /usr/bin/Xvfb :1 -screen 0 15360x8640x24 -dpi 96 +extension COMPOSITE +extension DAMAGE +extension GLX +extension RANDR +extension RENDER +extension MIT-SHM +extension XFIXES +extension XTEST +iglx +render -nolisten tcp -ac -noreset -shmem
root        8339  4.0  1.8 1320308 555536 ?      Ssl  3月26 143:50 kube-apiserver --advertise-address=192.168.18.133 --allow-privileged=true --audit-log-format=json --audit-log-maxage=7 --audit-log-maxbackup=10 --audit-log-maxsize=100 --audit-log-path=/var/log/kubernetes/audit.log --audit-policy-file=/etc/kubernetes/audit-policy.yml --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-aggregator-routing=true --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --feature-gates= --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/22 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
hanwang  3823945  0.4  1.6 7008400 486596 pts/4  Sl+  3月28   2:45 /home/hanwang/work/cowpaw/venv/bin/python3 /home/hanwang/work/cowpaw/venv/bin/copaw app --host 0.0.0.0
pcp         5294  0.4  1.5 2221828 469296 ?      Ssl  3月26  15:49 mysqld
pcp        13524  0.4  1.5 2353940 467044 ?      Ssl  3月26  16:50 mysqld
hanwang     7963 38.0  1.4 3689060 427180 pts/0  SLsl+ 3月26 1346:21 /usr/bin/retroarch -f
pcp         5574  0.5  1.3 2331232 388364 ?      Ssl  3月26  19:10 mysqld --max_connections=1000 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --default-authentication-plugin=mysql_native_password --tls_version=TLSv1.2,TLSv1.3 --init-file /data/application/init.sql --binlog_expire_logs_seconds=604800
hanwang  2764530  3.4  1.1 34785652 333112 ?     Sl   3月28  49:18 /home/hanwang/.qoder-server/bin/16c1dec9a3994f378353c6947b77c2dcf96e7c49/node --dns-result-order=ipv4first /home/hanwang/.qoder-server/bin/16c1dec9a3994f378353c6947b77c2dcf96e7c49/out/bootstrap-fork --type=extensionHost --transformURIs --useHostProxy=false
hanwang     6566  0.0  0.9 6587772 292244 ?      Sl   3月26   0:14 /snap/snap-store/1216/usr/bin/snap-store --gapplication-service
hanwang     4359  0.0  0.9 5169488 275436 ?      Ssl  3月26   2:57 /usr/bin/gnome-shell
root        5253  0.0  0.8 273463132 264468 ?    Ssl  3月26   0:47 /jellyfin/jellyfin
hanwang    17060  0.0  0.8 879348 253756 ?       SNl  3月26   2:42 /usr/bin/python3 /usr/bin/update-manager --no-update --no-focus-on-map
472         5575  0.3  0.8 1804476 244240 ?      Ssl  3月26  13:32 grafana server --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini --packaging=docker cfg:default.log.mode=console cfg:default.paths.data=/var/lib/grafana cfg:default.paths.logs=/var/log/grafana cfg:default.paths.plugins=/var/lib/grafana/plugins cfg:default.paths.provisioning=/etc/grafana/provisioning
hanwang  2764730  0.3  0.7 5805520 216720 ?      Sl   3月28   5:22 /home/hanwang/.qoder-server/bin/16c1dec9a3994f378353c6947b77c2dcf96e7c49/extensions/aicoding-agent/bin/x86_64_linux/Qoder start --workDir /home/hanwang/.config/Qoder/4fd368a50cab7edf1f493950ad5427e90b9ac12fca180ac55f52481aaf7b6541/SharedClientCache
root        1078  0.2  0.6 1638684 199424 ?      S<Lsl 3月26   7:45 ovs-vswitchd unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach
root         424  0.0  0.6 252252 178004 ?       S<s  3月26   0:30 /usr/lib/systemd/systemd-journald

2 问题发现

进程	PID	%MEM	RSS (实际内存)	说明
qemu-system-x86_64 (devstack-vm)	4186	28.3%	8.0 GB	KVM 虚拟机
Elasticsearch	8557	15.3%	4.3 GB	搜索引擎
opencode-ai	625102	2.0%	616 MB	AI 编码助手
Xvfb	7571	1.9%	585 MB	虚拟显示服务器
kube-apiserver	8339	1.8%	555 MB	K8s API 服务器
cowpaw (Python)	3823945	1.6%	486 MB	Python 应用
MySQL (3个实例)	5294/13524/5574	4.3%	共 ~1.3 GB	数据库
retroarch	7963	1.4%	427 MB	游戏模拟器
qoder-server (Node)	2764530	1.1%	333 MB	代码服务器

3、目前前两位服务暂时不用，考虑关掉

sudo virsh shutdown devstack-vm  # 直接关机释放 8GB，需要等待一段事件

4、总结了一套诊断脚本

#!/bin/bash
#===============================================================================
#
#          FILE: system_diagnosis.sh
#
#         USAGE: ./system_diagnosis.sh [options]
#
#   DESCRIPTION: 服务器卡顿分层排查脚本
#                1. 快速概览层 (1分钟)
#                2. 系统资源层 (CPU/内存/磁盘/网络)
#                3. 进程服务层 (TOP进程/僵尸进程/服务状态)
#                4. 深度分析层 (IO延迟/系统调用/硬件错误)
#
#       OPTIONS: -q (快速模式), -d (深度模式), -o <file> (输出到文件)
#  REQUIREMENTS: root权限(部分功能), sysstat/iperf3等工具(可选)
#          BUGS: 报告至 hanwang@example.com
#         NOTES: 建议在卡顿期间运行，多次采样更准确
#        AUTHOR: Han Wang, Cloud Platform Architect
#       VERSION: 2.0
#       CREATED: 2026-03-29
#===============================================================================

set -o pipefail

# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color

# 全局变量
REPORT_FILE=""
QUICK_MODE=false
DEEP_MODE=false
SAMPLE_INTERVAL=1
SAMPLE_COUNT=3
START_TIME=$(date +%s)

# 日志函数
log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

log_section() {
    echo -e "\n${CYAN}========================================${NC}"
    echo -e "${CYAN}  $1${NC}"
    echo -e "${CYAN}========================================${NC}\n"
}

# 检查root权限
check_root() {
    if [[ $EUID -ne 0 ]]; then
        log_warn "当前非root用户，部分诊断信息可能无法获取"
        log_warn "建议: sudo ./system_diagnosis.sh"
        echo ""
    fi
}

# 检查依赖工具
check_dependencies() {
    local missing_tools=()
    
    for tool in vmstat iostat mpstat pidstat ss; do
        if ! command -v $tool &> /dev/null; then
            missing_tools+=($tool)
        fi
    done
    
    if [[ ${#missing_tools[@]} -gt 0 ]]; then
        log_warn "缺少工具: ${missing_tools[*]}"
        log_info "安装: sudo apt-get install -y sysstat iproute2"
        echo ""
    fi
}

# 打印报告头部
print_header() {
    cat << 'EOF'
    ____  _   _ _____    _     _       _             
   / ___|| | | | ____|  | |   (_)_ __ | |_ ___  _ __ 
   \___ \| |_| |  _|    | |   | | '_ \| __/ _ \| '__|
    ___) |  _  | |___   | |___| | | | | || (_) | |   
   |____/|_| |_|_____|  |_____|_|_| |_|\__\___/|_|   
                                                     
   服务器卡顿分层排查脚本 v2.0
   Author: Han Wang | Cloud Platform Architect
EOF
    echo ""
    log_info "开始时间: $(date '+%Y-%m-%d %H:%M:%S')"
    log_info "主机名: $(hostname)"
    log_info "系统: $(lsb_release -ds 2>/dev/null || cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)"
    log_info "内核: $(uname -r)"
    log_info "架构: $(uname -m)"
    echo ""
}

#===============================================================================
# 第一层：快速概览 (1分钟健康检查)
#===============================================================================
layer1_quick_overview() {
    log_section "第一层：快速概览 (1分钟健康检查)"
    
    # 系统负载
    echo -e "${BLUE}[系统负载]${NC}"
    uptime_info=$(uptime)
    echo "  $uptime_info"
    
    # 解析负载值
    load_1min=$(echo "$uptime_info" | awk -F'load average:' '{print $2}' | awk -F',' '{print $1}' | tr -d ' ')
    cpu_cores=$(nproc)
    load_threshold=$(echo "$cpu_cores * 2" | bc -l 2>/dev/null || echo "$((cpu_cores * 2))")
    
    if [[ $(echo "$load_1min > $load_threshold" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
        log_error "1分钟负载 $load_1min 超过阈值 $load_threshold (CPU核数×2)，系统过载！"
    elif [[ $(echo "$load_1min > $cpu_cores" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
        log_warn "1分钟负载 $load_1min 超过CPU核数 $cpu_cores，负载较高"
    else
        log_info "负载正常: $load_1min / $cpu_cores 核"
    fi
    echo ""
    
    # 内存概览
    echo -e "${BLUE}[内存概览]${NC}"
    free -h | grep -E "(Mem|Swap)" | while read line; do
        echo "  $line"
    done
    
    mem_info=$(free | grep Mem)
    total_mem=$(echo $mem_info | awk '{print $2}')
    used_mem=$(echo $mem_info | awk '{print $3}')
    avail_mem=$(echo $mem_info | awk '{print $7}')
    mem_usage=$(echo "scale=1; $used_mem * 100 / $total_mem" | bc -l 2>/dev/null || echo "0")
    
    if [[ $(echo "$mem_usage > 90" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
        log_error "内存使用率 ${mem_usage}%，严重不足！"
    elif [[ $(echo "$mem_usage > 70" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
        log_warn "内存使用率 ${mem_usage}%，压力较大"
    else
        log_info "内存使用率 ${mem_usage}%，状态良好"
    fi
    
    # Swap检查
    swap_info=$(free | grep Swap)
    swap_total=$(echo $swap_info | awk '{print $2}')
    swap_used=$(echo $swap_info | awk '{print $3}')
    if [[ $swap_total -gt 0 && $swap_used -gt 0 ]]; then
        swap_usage=$(echo "scale=1; $swap_used * 100 / $swap_total" | bc -l 2>/dev/null || echo "0")
        if [[ $(echo "$swap_usage > 50" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            log_error "Swap使用率 ${swap_usage}%，频繁换页导致卡顿！"
        else
            log_warn "Swap使用率 ${swap_usage}%，注意内存压力"
        fi
    fi
    echo ""
    
    # 磁盘概览
    echo -e "${BLUE}[磁盘概览]${NC}"
    df -h | grep -E "^/dev" | while read line; do
        usage=$(echo $line | awk '{print $5}' | tr -d '%')
        mount=$(echo $line | awk '{print $6}')
        if [[ $usage -gt 90 ]]; then
            echo -e "  ${RED}[警告]${NC} $line"
        elif [[ $usage -gt 80 ]]; then
            echo -e "  ${YELLOW}[注意]${NC} $line"
        else
            echo "  $line"
        fi
    done
    echo ""
}

#===============================================================================
# 第二层：系统资源层详细检查
#===============================================================================
layer2_system_resources() {
    log_section "第二层：系统资源层详细检查"
    
    # CPU 详细分析
    echo -e "${BLUE}[CPU 详细分析]${NC}"
    echo "  物理核数: $(nproc)"
    echo "  型号: $(grep 'model name' /proc/cpuinfo | head -1 | cut -d':' -f2 | sed 's/^[ \t]*//')"
    
    if command -v mpstat &> /dev/null; then
        echo ""
        echo "  各核负载分布 (mpstat -P ALL):"
        mpstat -P ALL $SAMPLE_INTERVAL 1 | tail -n +4 | head -n 20 | while read line; do
            echo "    $line"
        done
        
        # 检查是否单核瓶颈
        max_cpu=$(mpstat -P ALL $SAMPLE_INTERVAL 1 | tail -n +4 | awk '{print $4}' | sort -rn | head -1)
        if [[ $(echo "$max_cpu > 80" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            log_warn "检测到单核使用率超过80%，可能存在单线程瓶颈"
        fi
    fi
    
    # 检查 steal time (虚拟机场景)
    if grep -q "st" /proc/stat 2>/dev/null; then
        steal_time=$(top -bn1 | grep "Cpu(s)" | grep -oP '\d+\.\d+(?=\s*st)' | head -1)
        if [[ -n "$steal_time" && $(echo "$steal_time > 5" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            log_error "CPU Steal Time: ${steal_time}%，宿主机超售严重！"
        fi
    fi
    echo ""
    
    # 内存压力检测
    echo -e "${BLUE}[内存压力检测]${NC}"
    
    # /proc/meminfo 关键指标
    echo "  关键指标:"
    for key in MemTotal MemFree MemAvailable Buffers Cached Active Inactive Dirty Writeback AnonPages; do
        value=$(grep "^$key:" /proc/meminfo | awk '{print $2, $3}')
        [[ -n "$value" ]] && echo "    $key: $value"
    done
    
    # Dirty 内存检查
    dirty=$(grep "^Dirty:" /proc/meminfo | awk '{print $2}')
    if [[ $dirty -gt 102400 ]]; then  # > 100MB
        log_warn "Dirty内存 ${dirty}KB，大量数据待写入磁盘，即将触发IO风暴"
    fi
    
    # 检查 OOM 历史
    if dmesg 2>/dev/null | grep -i "out of memory" | tail -5 | grep -q "oom"; then
        log_error "检测到历史 OOM 事件！"
        echo "  最近5条:"
        dmesg 2>/dev/null | grep -i "out of memory" | tail -5 | sed 's/^/    /'
    fi
    echo ""
    
    # 磁盘 IO 分析
    echo -e "${BLUE}[磁盘 IO 分析]${NC}"
    if command -v iostat &> /dev/null; then
        echo "  IO统计 (iostat -xz):"
        iostat -xz $SAMPLE_INTERVAL 1 | tail -n +4 | head -20 | while read line; do
            echo "    $line"
        done
        
        # 提取关键指标
        disk_util=$(iostat -xz $SAMPLE_INTERVAL 1 2>/dev/null | awk '/^nvme|^sd|^vd/{getline; print $NF}' | sort -rn | head -1)
        if [[ -n "$disk_util" && $(echo "$disk_util > 90" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            log_error "磁盘利用率 ${disk_util}%，IO 饱和！"
        elif [[ -n "$disk_util" && $(echo "$disk_util > 70" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            log_warn "磁盘利用率 ${disk_util}%，IO 压力较大"
        fi
    else
        log_warn "未安装 sysstat，无法获取 IO 统计"
    fi
    
    # 检查磁盘延迟 (通过 /proc/diskstats)
    echo ""
    echo "  磁盘延迟估算 (读取/写入 ms):"
    cat /proc/diskstats | grep -E "nvme|sd[a-z] " | while read line; do
        device=$(echo $line | awk '{print $3}')
        reads=$(echo $line | awk '{print $6}')
        read_ms=$(echo $line | awk '{print $7}')
        writes=$(echo $line | awk '{print $10}')
        write_ms=$(echo $line | awk '{print $11}')
        
        if [[ $reads -gt 0 ]]; then
            avg_read_latency=$(echo "scale=2; $read_ms / $reads" | bc -l 2>/dev/null || echo "N/A")
        else
            avg_read_latency="N/A"
        fi
        
        if [[ $writes -gt 0 ]]; then
            avg_write_latency=$(echo "scale=2; $write_ms / $writes" | bc -l 2>/dev/null || echo "N/A")
        else
            avg_write_latency="N/A"
        fi
        
        echo "    $device: 读延迟=${avg_read_latency}ms, 写延迟=${avg_write_latency}ms"
    done
    echo ""
    
    # 网络分析
    echo -e "${BLUE}[网络连接分析]${NC}"
    echo "  连接状态统计:"
    ss -s 2>/dev/null | sed 's/^/    /'
    
    echo ""
    echo "  TCP 状态分布:"
    ss -ant | awk 'NR>1 {++S[$1]} END {for(a in S) print "    " a, S[a]}' | sort -rn -k2
    
    # 检查 TIME_WAIT 和 CLOSE_WAIT
    time_wait=$(ss -ant | grep -c TIME_WAIT 2>/dev/null || echo "0")
    close_wait=$(ss -ant | grep -c CLOSE_WAIT 2>/dev/null || echo "0")
    
    if [[ $close_wait -gt 100 ]]; then
        log_error "CLOSE_WAIT 连接数 $close_wait，存在连接泄漏！"
    fi
    
    if [[ $time_wait -gt 10000 ]]; then
        log_warn "TIME_WAIT 连接数 $time_wait，端口可能耗尽"
    fi
    echo ""
}

#===============================================================================
# 第三层：进程服务层检查
#===============================================================================
layer3_process_services() {
    log_section "第三层：进程服务层检查"
    
    # TOP 资源消耗进程
    echo -e "${BLUE}[TOP 资源消耗进程]${NC}"
    
    echo "  CPU 占用 TOP 10:"
    ps aux --sort=-%cpu | head -11 | tail -10 | nl -w 2 -s '. ' | while read line; do
        pid=$(echo $line | awk '{print $3}')
        cpu=$(echo $line | awk '{print $4}')
        mem=$(echo $line | awk '{print $5}')
        cmd=$(echo $line | cut -d' ' -f12-)
        
        if [[ $(echo "$cpu > 50" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            echo -e "    ${RED}$line${NC}"
        elif [[ $(echo "$cpu > 20" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            echo -e "    ${YELLOW}$line${NC}"
        else
            echo "    $line"
        fi
    done
    
    echo ""
    echo "  内存占用 TOP 10:"
    ps aux --sort=-%mem | head -11 | tail -10 | nl -w 2 -s '. ' | while read line; do
        mem=$(echo $line | awk '{print $5}')
        if [[ $(echo "$mem > 10" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            echo -e "    ${RED}$line${NC}"
        elif [[ $(echo "$mem > 5" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
            echo -e "    ${YELLOW}$line${NC}"
        else
            echo "    $line"
        fi
    done
    echo ""
    
    # 僵尸进程检查
    echo -e "${BLUE}[僵尸进程检查]${NC}"
    zombie_count=$(ps aux | awk '$8=="Z" {count++} END {print count+0}')
    if [[ $zombie_count -gt 0 ]]; then
        log_error "发现 $zombie_count 个僵尸进程！"
        echo "  僵尸进程列表:"
        ps aux | awk '$8=="Z" {print "    PID: " $2, "PPID: " $3, "CMD: " $11}' | head -10
        echo "  父进程:"
        ps aux | awk '$8=="Z" {print $3}' | sort -u | xargs -I {} ps -p {} -o pid,comm 2>/dev/null
    else
        log_info "未发现僵尸进程"
    fi
    echo ""
    
    # D 状态进程（不可中断睡眠，通常是IO等待）
    echo -e "${BLUE}[D 状态进程检查 (不可中断睡眠)]${NC}"
    d_state_procs=$(ps aux | awk '$8=="D" {print $0}')
    if [[ -n "$d_state_procs" ]]; then
        log_error "发现 D 状态进程（IO 等待/死锁）:"
        echo "$d_state_procs" | head -5 | sed 's/^/    /'
    else
        log_info "未发现 D 状态进程"
    fi
    echo ""
    
    # 系统服务状态
    echo -e "${BLUE}[关键服务状态]${NC}"
    critical_services=("sshd" "systemd-journald" "cron" "networking" "docker" "containerd" "kubelet")
    for service in "${critical_services[@]}"; do
        if systemctl is-active --quiet $service 2>/dev/null; then
            echo "  [运行中] $service"
        elif systemctl is-failed --quiet $service 2>/dev/null; then
            echo -e "  ${RED}[失败] $service${NC}"
        else
            echo "  [未运行/未安装] $service"
        fi
    done
    echo ""
    
    # 定时任务检查
    echo -e "${BLUE}[定时任务检查]${NC}"
    echo "  当前时间: $(date '+%H:%M')"
    echo "  最近5分钟内可能执行的 cron 任务:"
    crontab -l 2>/dev/null | grep -v "^#" | while read line; do
        echo "    $line"
    done
    ls -la /etc/cron.d/ 2>/dev/null | tail -n +3 | while read line; do
        echo "    /etc/cron.d/$(echo $line | awk '{print $9}')"
    done
    echo ""
}

#===============================================================================
# 第四层：深度分析（可选）
#===============================================================================
layer4_deep_analysis() {
    if [[ "$DEEP_MODE" != true ]]; then
        return
    fi
    
    log_section "第四层：深度分析 (Deep Mode)"
    
    # IO 压力详细分析
    echo -e "${BLUE}[IO 压力详细分析]${NC}"
    if [[ -f /proc/pressure/io ]]; then
        echo "  PSI IO 压力统计:"
        cat /proc/pressure/io | sed 's/^/    /'
    fi
    
    if [[ -f /proc/pressure/memory ]]; then
        echo ""
        echo "  PSI 内存压力统计:"
        cat /proc/pressure/memory | sed 's/^/    /'
    fi
    echo ""
    
    # 系统调用跟踪（采样）
    echo -e "${BLUE}[系统调用采样 (pidstat)]${NC}"
    if command -v pidstat &> /dev/null; then
        echo "  采样 5 秒，查看系统调用频率..."
        pidstat -S 1 5 2>/dev/null | tail -20 | sed 's/^/    /'
    fi
    echo ""
    
    # 上下文切换和中断
    echo -e "${BLUE}[上下文切换统计]${NC}"
    if command -v vmstat &> /dev/null; then
        echo "  (cs: 上下文切换, in: 中断, us/sy/id/wa: CPU时间分布)"
        vmstat -s | grep -E "(context switch|interrupt|CPU)" | sed 's/^/    /'
    fi
    echo ""
    
    # 硬件错误检查
    echo -e "${BLUE}[硬件错误检查]${NC}"
    if command -v mcelog &> /dev/null; then
        mcelog --client 2>/dev/null | head -10 | sed 's/^/    /'
    else
        log_info "未安装 mcelog，跳过硬件错误检查"
    fi
    
    # 内核日志错误
    echo ""
    echo "  最近内核错误 (dmesg):"
    dmesg -T 2>/dev/null | grep -iE "(error|fail|warn|oom|killed)" | tail -10 | sed 's/^/    /'
    echo ""
}

#===============================================================================
# 诊断报告生成
#===============================================================================
generate_report() {
    log_section "诊断报告总结"
    
    END_TIME=$(date +%s)
    DURATION=$((END_TIME - START_TIME))
    
    echo "排查耗时: ${DURATION} 秒"
    echo ""
    
    # 生成建议
    echo -e "${BLUE}[优化建议]${NC}"
    
    # 根据收集的信息给出建议
    load=$(uptime | awk -F'load average:' '{print $2}' | awk -F',' '{print $1}' | tr -d ' ')
    mem_usage=$(free | grep Mem | awk '{printf("%.0f", $3/$2 * 100)}')
    
    if [[ $(echo "$load > $(nproc)" | bc -l 2>/dev/null || echo "0") -eq 1 ]]; then
        echo "  1. [高负载] 系统负载过高，建议:"
        echo "     - 检查 CPU 占用最高的进程，考虑优化或迁移"
        echo "     - 如果是多核系统，检查是否存在单线程瓶颈"
    fi
    
    if [[ $mem_usage -gt 80 ]]; then
        echo "  2. [内存不足] 内存使用率 ${mem_usage}%，建议:"
        echo "     - 找出内存占用 TOP 进程，关闭不必要的服务"
        echo "     - 检查是否有内存泄漏（RSS 持续增长的进程）"
        echo "     - 考虑增加物理内存或启用 swap（临时方案）"
    fi
    
    # 检查是否有僵尸进程
    zombie_count=$(ps aux | awk '$8=="Z" {count++} END {print count+0}')
    if [[ $zombie_count -gt 0 ]]; then
        echo "  3. [僵尸进程] 存在 $zombie_count 个僵尸进程，建议重启父进程或系统"
    fi
    
    # 检查 D 状态进程
    d_count=$(ps aux | awk '$8=="D" {count++} END {print count+0}')
    if [[ $d_count -gt 0 ]]; then
        echo "  4. [IO 阻塞] 存在 $d_count 个 D 状态进程，建议:"
        echo "     - 检查磁盘健康状态 (smartctl)"
        echo "     - 检查是否有 NFS/iSCSI 等网络存储挂死"
    fi
    
    echo ""
    echo -e "${GREEN}诊断完成！如需进一步分析，请查看上述详细输出。${NC}"
}

#===============================================================================
# 主函数
#===============================================================================
main() {
    # 参数解析
    while getopts "qdo:h" opt; do
        case $opt in
            q) QUICK_MODE=true ;;
            d) DEEP_MODE=true ;;
            o) REPORT_FILE="$OPTARG" ;;
            h|*) 
                echo "Usage: $0 [-q] [-d] [-o <output_file>]"
                echo "  -q    快速模式 (仅第一层)"
                echo "  -d    深度模式 (包含第四层)"
                echo "  -o    输出到文件"
                exit 0
                ;;
        esac
    done
    
    # 重定向输出到文件
    if [[ -n "$REPORT_FILE" ]]; then
        exec > >(tee -a "$REPORT_FILE")
        exec 2>&1
    fi
    
    # 执行检查
    check_root
    check_dependencies
    print_header
    
    # 分层执行
    layer1_quick_overview
    
    if [[ "$QUICK_MODE" != true ]]; then
        layer2_system_resources
        layer3_process_services
        layer4_deep_analysis
    fi
    
    generate_report
}

# 运行主函数
main "$@"

执行

bash diagnose.sh