watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [swapper/0:1]

测试代码如下：

cpp 复制代码

#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>

static DEFINE_SPINLOCK(hack_spinA);
static DEFINE_SPINLOCK(hack_spinB);

void hack_spinAB(void)
{
    printk("hack_lockdep:A->B\n");
    spin_lock(&hack_spinA);
    spin_lock(&hack_spinB);
}

void hack_spinBA(void)
{
    printk("hack_lockdep:B->A\n");
    spin_lock(&hack_spinB);
}

static int __init lockdep_test_init(void)
{
    printk("figo:my lockdep module init\n");
    
    hack_spinAB();
    hack_spinBA();
     
    return 0;
}
    
    static void __exit lockdep_test_exit(void)
    {
      printk("goodbye\n");
    }
    
    module_init(lockdep_test_init);
    module_exit(lockdep_test_exit);
    MODULE_LICENSE("GPL");

$28.015539\] -(3)\[1:swapper/0-\>0:swapper/0\]watchdog: BUG: soft lockup - CPU#3 stuck for 23s! \[swapper/0:1$ $28.025300\] -(3)\[1:swapper/0-\>0:swapper/0\]Kernel panic - not syncing: softlockup: hung tasks \[ 28.033776\] -(3)\[1:swapper/0-\>0:swapper/0\]CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2 \[ 28.043906\] -(3)\[1:swapper/0-\>0:swapper/0\]Hardware name: Platfrom(DT) \[ 28.050636\] -(3)\[1:swapper/0-\>0:swapper/0\]Call trace: \[ 28.055711\] -(3)\[1:swapper/0-\>0:swapper/0\] dump_backtrace+0x0/0x198 \[ 28.062006\] -(3)\[1:swapper/0-\>0:swapper/0\] show_stack+0x20/0x2c \[ 28.067951\] -(3)\[1:swapper/0-\>0:swapper/0\] dump_stack+0xd8/0x128 \[ 28.073984\] -(3)\[1:swapper/0-\>0:swapper/0\] panic+0x134/0x334 \[ 28.079669\] -(3)\[1:swapper/0-\>0:swapper/0\] softlockup_fn+0x0/0x60 \[ 28.085790\] -(3)\[1:swapper/0-\>0:swapper/0\] __run_hrtimer+0xa8/0x2b0 \[ 28.092085\] -(3)\[1:swapper/0-\>0:swapper/0\] hrtimer_interrupt+0x174/0x3c8 \[ 28.098817\] -(3)\[1:swapper/0-\>0:swapper/0\] arch_timer_handler_phys+0x40/0x50 \[ 28.105897\] -(3)\[1:swapper/0-\>0:swapper/0\] handle_percpu_devid_irq+0x88/0x278 \[ 28.113065\] -(3)\[1:swapper/0-\>0:swapper/0\] __handle_domain_irq+0x84/0xc4 \[ 28.119796\] -(3)\[1:swapper/0-\>0:swapper/0\] gic_handle_irq+0x10c/0x184 \[ 28.126265\] -(3)\[1:swapper/0-\>0:swapper/0\] el1_irq+0xec/0x198 \[ 28.132035\] -(3)\[1:swapper/0-\>0:swapper/0\] queued_spin_lock_slowpath+0x94/0x270 \[ 28.139376\] -(3)\[1:swapper/0-\>0:swapper/0\] _raw_spin_lock+0x50/0x54 \[ 28.145670\] -(3)\[1:swapper/0-\>0:swapper/0\] lockdep_test_init+0x34/0x40 \[ 28.152226\] -(3)\[1:swapper/0-\>0:swapper/0\] do_one_initcall+0xa0/0x298 \[ 28.158694\] -(3)\[1:swapper/0-\>0:swapper/0\] do_initcall_level+0xc0/0x164 \[ 28.165337\] -(3)\[1:swapper/0-\>0:swapper/0\] do_basic_setup+0x34/0x4c \[ 28.171631\] -(3)\[1:swapper/0-\>0:swapper/0\] kernel_init_freeable+0xd0/0x150 \[ 28.178535\] -(3)\[1:swapper/0-\>0:swapper/0\] kernel_init+0x18/0x29c \[ 28.184655\] -(3)\[1:swapper/0-\>0:swapper/0\] ret_from_fork+0x10/0x18 \[ 28.190864\] -(3)\[1:swapper/0-\>0:swapper/0\]SMP: stopping secondary CPUs \[ 28.197425\] -(3)\[1:swapper/0-\>0:swapper/0\]Kernel Offset: disabled \[ 28.203545\] -(3)\[1:swapper/0-\>0:swapper/0\]CPU features: 0x00000000,22002a38 \[ 28.210537\] -(3)\[1:swapper/0-\>0:swapper/0\]Memory Limit: none \[ 28.216221\] -(3)\[1:swapper/0-\>0:swapper/0\]eskdump die \[ 28.221295\] -(3)\[1:swapper/0-\>0:swapper/0\]CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2 \[ 28.231425\] -(3)\[1:swapper/0-\>0:swapper/0\]Hardware name: ES (DT) \[ 28.238156\] -(3)\[1:swapper/0-\>0:swapper/0\]pstate: 60000089 (nZCv daIf -PAN -UAO) \[ 28.245584\] -(3)\[1:swapper/0-\>0:swapper/0\]pc : panic+0x1b8/0x334 \[ 28.251617\] -(3)\[1:swapper/0-\>0:swapper/0\]lr : panic+0x148/0x334 \[ 28.257649\] -(3)\[1:swapper/0-\>0:swapper/0\]sp : ffff00000801bbe0 \[ 28.263592\] -(3)\[1:swapper/0-\>0:swapper/0\]x29: ffff00000801bd80 x28: 0000000000000001 \[ 28.271543\] -(3)\[1:swapper/0-\>0:swapper/0\]x27: ffff80037ff9d180 x26: 0000000685682d4d \[ 28.279494\] -(3)\[1:swapper/0-\>0:swapper/0\]x25: ffff00000a4fd000 x24: 0000000000000003 \[ 28.287445\] -(3)\[1:swapper/0-\>0:swapper/0\]x23: ffff00000a0208b0 x22: 0000000000000000 \[ 28.295396\] -(3)\[1:swapper/0-\>0:swapper/0\]x21: ffff800379219ec0 x20: 0000000000000000 \[ 28.303346\] -(3)\[1:swapper/0-\>0:swapper/0\]x19: ffff000009af821d x18: 0000000000000001 \[ 28.311297\] -(3)\[1:swapper/0-\>0:swapper/0\]x17: 00000000000cc70c x16: 00000000000000f4 \[ 28.319248\] -(3)\[1:swapper/0-\>0:swapper/0\]x15: ffff00000923f160 x14: ffff00000980e250 \[ 28.327199\] -(3)\[1:swapper/0-\>0:swapper/0\]x13: 000000000000369a x12: 0000000029aaaaab \[ 28.335150\] -(3)\[1:swapper/0-\>0:swapper/0\]x11: 0000000000000001 x10: 0000000006666998 \[ 28.343100\] -(3)\[1:swapper/0-\>0:swapper/0\]x9 : 4b602900c22d1900 x8 : ffff00000801bc20 \[ 28.351051\] -(3)\[1:swapper/0-\>0:swapper/0\]x7 : 73203a504d535d30 x6 : ffff00000a8b5b16 \[ 28.359002\] -(3)\[1:swapper/0-\>0:swapper/0\]x5 : 0000000000000001 x4 : 000000000000000c \[ 28.366953\] -(3)\[1:swapper/0-\>0:swapper/0\]x3 : 000000000000004a x2 : 0000000000000003 \[ 28.374903\] -(3)\[1:swapper/0-\>0:swapper/0\]x1 : 0000000000000004 x0 : 0000000000000001 \[ 28.382854\] -(3)\[1:swapper/0-\>0:swapper/0\]Modules linked in: \[ 28.397593\] -(3)\[1:swapper/0-\>0:swapper/0\]done available data size 0x83000 \[ 28.404503\] -(3)\[1:swapper/0-\>0:swapper/0\]kdump_sign sig 0x8a6acdb4, size 0x83000 \[ 28.412017\] -(3)\[1:swapper/0-\>0:swapper/0\]rebooting... 这里没有在kernel 添加lockdep配置，打印的calltrace信息，测试代码逻辑比较简单，所以能一眼看出来lockdep_test_init中有问题$

Linux内核中打开lockdep功能，需要打开CONFIG_DEBUG_LOCKDEP选项：

cpp 复制代码

CONFIG_LOCK_STAT=y
CONFIG_PROVE_LOCKING=y
CONFIG_DEBUG_LOCKDEP=y

cpp 复制代码

console:/ # insmod ./lockdep_test.ko
[  501.016783] -(3)[2844:insmod->2519:sh]
[  501.020560] -(3)[2844:insmod->2519:sh]============================================
[  501.028164] -(3)[2844:insmod->2519:sh]WARNING: possible recursive locking detected
[  501.035768] -(3)[2844:insmod->2519:sh]4.19.176 #1 Tainted: G           O
[  501.042935] -(3)[2844:insmod->2519:sh]--------------------------------------------
[  501.050538] -(3)[2844:insmod->2519:sh]insmod/2844 is trying to acquire lock:
[  501.057617] 00000000f287eb44 (hack_spinB){+.+.}, at: init_module+0x34/0xfb0 [lockdep_test]
[  501.065931] -(3)[2844:insmod->2519:sh]
[  501.065931] but task is already holding lock:
[  501.074056] 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[  501.082275] -(3)[2844:insmod->2519:sh]
[  501.082275] other info that might help us debug this:
[  501.091100] -(3)[2844:insmod->2519:sh] Possible unsafe locking scenario:
[  501.091100]
[  501.099313] -(3)[2844:insmod->2519:sh]       CPU0
[  501.104038] -(3)[2844:insmod->2519:sh]       ----
[  501.108761] -(3)[2844:insmod->2519:sh]  lock(hack_spinB);
[  501.114184] -(3)[2844:insmod->2519:sh]  lock(hack_spinB);
[  501.119608] -(3)[2844:insmod->2519:sh]
[  501.119608]  *** DEADLOCK ***
[  501.119608]
[  501.127823] -(3)[2844:insmod->2519:sh] May be due to missing lock nesting notation
[  501.127823]
[  501.136908] -(3)[2844:insmod->2519:sh]2 locks held by insmod/2844:
[  501.143114] -(3)[2844:insmod->2519:sh] #0: 000000000721ea50 (hack_spinA){+.+.}, at: hack_spinAB+0x24/0x38 [lockdep_test]
[  501.154035] -(3)[2844:insmod->2519:sh] #1: 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[  501.164958] -(3)[2844:insmod->2519:sh]
[  501.164958] stack backtrace:
[  501.171605] -(3)[2844:insmod->2519:sh]CPU: 3 PID: 2844 Comm: insmod Tainted: G           O      4.19.176 #1
[  501.181388] -(3)[2844:insmod->2519:sh]Hardware name: ESWIN_LD60 (DT)
[  501.187770] -(3)[2844:insmod->2519:sh]Call trace:
[  501.192500] -(3)[2844:insmod->2519:sh] dump_backtrace+0x0/0x198
[  501.198445] -(3)[2844:insmod->2519:sh] show_stack+0x20/0x2c
[  501.204045] -(3)[2844:insmod->2519:sh] dump_stack+0xd8/0x128
[  501.209732] -(3)[2844:insmod->2519:sh] __lock_acquire+0x17d8/0x1ed0
[  501.216028] -(3)[2844:insmod->2519:sh] lock_acquire+0x12c/0x1f4
[  501.221976] -(3)[2844:insmod->2519:sh] _raw_spin_lock+0x54/0x90
[  501.227922] -(3)[2844:insmod->2519:sh] init_module+0x34/0xfb0 [lockdep_test]
[  501.235003] -(3)[2844:insmod->2519:sh] do_one_initcall+0x13c/0x3e0
[  501.241213] -(3)[2844:insmod->2519:sh] do_init_module+0x5c/0x218
[  501.247248] -(3)[2844:insmod->2519:sh] load_module+0x296c/0x31d4
[  501.253283] -(3)[2844:insmod->2519:sh] __se_sys_finit_module+0xd8/0x114
[  501.259928] -(3)[2844:insmod->2519:sh] __arm64_sys_finit_module+0x20/0x2c
[  501.266748] -(3)[2844:insmod->2519:sh] el0_svc_common+0xa0/0x170
[  501.272782] -(3)[2844:insmod->2519:sh] el0_svc_compat_handler+0x28/0x34
[  501.279425] -(3)[2844:insmod->2519:sh] el0_svc_compat+0x8/0x34