watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [swapper/0:1]

测试代码如下：

cpp 复制代码

#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>

static DEFINE_SPINLOCK(hack_spinA);
static DEFINE_SPINLOCK(hack_spinB);

void hack_spinAB(void)
{
    printk("hack_lockdep:A->B\n");
    spin_lock(&hack_spinA);
    spin_lock(&hack_spinB);
}

void hack_spinBA(void)
{
    printk("hack_lockdep:B->A\n");
    spin_lock(&hack_spinB);
}

static int __init lockdep_test_init(void)
{
    printk("figo:my lockdep module init\n");
    
    hack_spinAB();
    hack_spinBA();
     
    return 0;
}
    
    static void __exit lockdep_test_exit(void)
    {
      printk("goodbye\n");
    }
    
    module_init(lockdep_test_init);
    module_exit(lockdep_test_exit);
    MODULE_LICENSE("GPL");

[ 28.015539] -(3)[1:swapper/0->0:swapper/0]watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [swapper/0:1]

[ 28.025300] -(3)[1:swapper/0->0:swapper/0]Kernel panic - not syncing: softlockup: hung tasks

[ 28.033776] -(3)[1:swapper/0->0:swapper/0]CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2

[ 28.043906] -(3)[1:swapper/0->0:swapper/0]Hardware name: Platfrom(DT)

[ 28.050636] -(3)[1:swapper/0->0:swapper/0]Call trace:

[ 28.055711] -(3)[1:swapper/0->0:swapper/0] dump_backtrace+0x0/0x198

[ 28.062006] -(3)[1:swapper/0->0:swapper/0] show_stack+0x20/0x2c

[ 28.067951] -(3)[1:swapper/0->0:swapper/0] dump_stack+0xd8/0x128

[ 28.073984] -(3)[1:swapper/0->0:swapper/0] panic+0x134/0x334

[ 28.079669] -(3)[1:swapper/0->0:swapper/0] softlockup_fn+0x0/0x60

[ 28.085790] -(3)[1:swapper/0->0:swapper/0] __run_hrtimer+0xa8/0x2b0

[ 28.092085] -(3)[1:swapper/0->0:swapper/0] hrtimer_interrupt+0x174/0x3c8

[ 28.098817] -(3)[1:swapper/0->0:swapper/0] arch_timer_handler_phys+0x40/0x50

[ 28.105897] -(3)[1:swapper/0->0:swapper/0] handle_percpu_devid_irq+0x88/0x278

[ 28.113065] -(3)[1:swapper/0->0:swapper/0] __handle_domain_irq+0x84/0xc4

[ 28.119796] -(3)[1:swapper/0->0:swapper/0] gic_handle_irq+0x10c/0x184

[ 28.126265] -(3)[1:swapper/0->0:swapper/0] el1_irq+0xec/0x198

[ 28.132035] -(3)[1:swapper/0->0:swapper/0] queued_spin_lock_slowpath+0x94/0x270

[ 28.139376] -(3)[1:swapper/0->0:swapper/0] _raw_spin_lock+0x50/0x54

[ 28.145670] -(3)[1:swapper/0->0:swapper/0] lockdep_test_init+0x34/0x40

[ 28.152226] -(3)[1:swapper/0->0:swapper/0] do_one_initcall+0xa0/0x298

[ 28.158694] -(3)[1:swapper/0->0:swapper/0] do_initcall_level+0xc0/0x164

[ 28.165337] -(3)[1:swapper/0->0:swapper/0] do_basic_setup+0x34/0x4c

[ 28.171631] -(3)[1:swapper/0->0:swapper/0] kernel_init_freeable+0xd0/0x150

[ 28.178535] -(3)[1:swapper/0->0:swapper/0] kernel_init+0x18/0x29c

[ 28.184655] -(3)[1:swapper/0->0:swapper/0] ret_from_fork+0x10/0x18

[ 28.190864] -(3)[1:swapper/0->0:swapper/0]SMP: stopping secondary CPUs

[ 28.197425] -(3)[1:swapper/0->0:swapper/0]Kernel Offset: disabled

[ 28.203545] -(3)[1:swapper/0->0:swapper/0]CPU features: 0x00000000,22002a38

[ 28.210537] -(3)[1:swapper/0->0:swapper/0]Memory Limit: none

[ 28.216221] -(3)[1:swapper/0->0:swapper/0]eskdump die

[ 28.221295] -(3)[1:swapper/0->0:swapper/0]CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2

[ 28.231425] -(3)[1:swapper/0->0:swapper/0]Hardware name: ES (DT)

[ 28.238156] -(3)[1:swapper/0->0:swapper/0]pstate: 60000089 (nZCv daIf -PAN -UAO)

[ 28.245584] -(3)[1:swapper/0->0:swapper/0]pc : panic+0x1b8/0x334

[ 28.251617] -(3)[1:swapper/0->0:swapper/0]lr : panic+0x148/0x334

[ 28.257649] -(3)[1:swapper/0->0:swapper/0]sp : ffff00000801bbe0

[ 28.263592] -(3)[1:swapper/0->0:swapper/0]x29: ffff00000801bd80 x28: 0000000000000001

[ 28.271543] -(3)[1:swapper/0->0:swapper/0]x27: ffff80037ff9d180 x26: 0000000685682d4d

[ 28.279494] -(3)[1:swapper/0->0:swapper/0]x25: ffff00000a4fd000 x24: 0000000000000003

[ 28.287445] -(3)[1:swapper/0->0:swapper/0]x23: ffff00000a0208b0 x22: 0000000000000000

[ 28.295396] -(3)[1:swapper/0->0:swapper/0]x21: ffff800379219ec0 x20: 0000000000000000

[ 28.303346] -(3)[1:swapper/0->0:swapper/0]x19: ffff000009af821d x18: 0000000000000001

[ 28.311297] -(3)[1:swapper/0->0:swapper/0]x17: 00000000000cc70c x16: 00000000000000f4

[ 28.319248] -(3)[1:swapper/0->0:swapper/0]x15: ffff00000923f160 x14: ffff00000980e250

[ 28.327199] -(3)[1:swapper/0->0:swapper/0]x13: 000000000000369a x12: 0000000029aaaaab

[ 28.335150] -(3)[1:swapper/0->0:swapper/0]x11: 0000000000000001 x10: 0000000006666998

[ 28.343100] -(3)[1:swapper/0->0:swapper/0]x9 : 4b602900c22d1900 x8 : ffff00000801bc20

[ 28.351051] -(3)[1:swapper/0->0:swapper/0]x7 : 73203a504d535d30 x6 : ffff00000a8b5b16

[ 28.359002] -(3)[1:swapper/0->0:swapper/0]x5 : 0000000000000001 x4 : 000000000000000c

[ 28.366953] -(3)[1:swapper/0->0:swapper/0]x3 : 000000000000004a x2 : 0000000000000003

[ 28.374903] -(3)[1:swapper/0->0:swapper/0]x1 : 0000000000000004 x0 : 0000000000000001

[ 28.382854] -(3)[1:swapper/0->0:swapper/0]Modules linked in:

[ 28.397593] -(3)[1:swapper/0->0:swapper/0]done available data size 0x83000

[ 28.404503] -(3)[1:swapper/0->0:swapper/0]kdump_sign sig 0x8a6acdb4, size 0x83000

[ 28.412017] -(3)[1:swapper/0->0:swapper/0]rebooting...
这里没有在kernel 添加lockdep配置，打印的calltrace信息，测试代码逻辑比较简单，所以能一眼看出来lockdep_test_init中有问题

Linux内核中打开lockdep功能，需要打开CONFIG_DEBUG_LOCKDEP选项：

cpp 复制代码

CONFIG_LOCK_STAT=y
CONFIG_PROVE_LOCKING=y
CONFIG_DEBUG_LOCKDEP=y

cpp 复制代码

console:/ # insmod ./lockdep_test.ko
[  501.016783] -(3)[2844:insmod->2519:sh]
[  501.020560] -(3)[2844:insmod->2519:sh]============================================
[  501.028164] -(3)[2844:insmod->2519:sh]WARNING: possible recursive locking detected
[  501.035768] -(3)[2844:insmod->2519:sh]4.19.176 #1 Tainted: G           O
[  501.042935] -(3)[2844:insmod->2519:sh]--------------------------------------------
[  501.050538] -(3)[2844:insmod->2519:sh]insmod/2844 is trying to acquire lock:
[  501.057617] 00000000f287eb44 (hack_spinB){+.+.}, at: init_module+0x34/0xfb0 [lockdep_test]
[  501.065931] -(3)[2844:insmod->2519:sh]
[  501.065931] but task is already holding lock:
[  501.074056] 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[  501.082275] -(3)[2844:insmod->2519:sh]
[  501.082275] other info that might help us debug this:
[  501.091100] -(3)[2844:insmod->2519:sh] Possible unsafe locking scenario:
[  501.091100]
[  501.099313] -(3)[2844:insmod->2519:sh]       CPU0
[  501.104038] -(3)[2844:insmod->2519:sh]       ----
[  501.108761] -(3)[2844:insmod->2519:sh]  lock(hack_spinB);
[  501.114184] -(3)[2844:insmod->2519:sh]  lock(hack_spinB);
[  501.119608] -(3)[2844:insmod->2519:sh]
[  501.119608]  *** DEADLOCK ***
[  501.119608]
[  501.127823] -(3)[2844:insmod->2519:sh] May be due to missing lock nesting notation
[  501.127823]
[  501.136908] -(3)[2844:insmod->2519:sh]2 locks held by insmod/2844:
[  501.143114] -(3)[2844:insmod->2519:sh] #0: 000000000721ea50 (hack_spinA){+.+.}, at: hack_spinAB+0x24/0x38 [lockdep_test]
[  501.154035] -(3)[2844:insmod->2519:sh] #1: 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[  501.164958] -(3)[2844:insmod->2519:sh]
[  501.164958] stack backtrace:
[  501.171605] -(3)[2844:insmod->2519:sh]CPU: 3 PID: 2844 Comm: insmod Tainted: G           O      4.19.176 #1
[  501.181388] -(3)[2844:insmod->2519:sh]Hardware name: ESWIN_LD60 (DT)
[  501.187770] -(3)[2844:insmod->2519:sh]Call trace:
[  501.192500] -(3)[2844:insmod->2519:sh] dump_backtrace+0x0/0x198
[  501.198445] -(3)[2844:insmod->2519:sh] show_stack+0x20/0x2c
[  501.204045] -(3)[2844:insmod->2519:sh] dump_stack+0xd8/0x128
[  501.209732] -(3)[2844:insmod->2519:sh] __lock_acquire+0x17d8/0x1ed0
[  501.216028] -(3)[2844:insmod->2519:sh] lock_acquire+0x12c/0x1f4
[  501.221976] -(3)[2844:insmod->2519:sh] _raw_spin_lock+0x54/0x90
[  501.227922] -(3)[2844:insmod->2519:sh] init_module+0x34/0xfb0 [lockdep_test]
[  501.235003] -(3)[2844:insmod->2519:sh] do_one_initcall+0x13c/0x3e0
[  501.241213] -(3)[2844:insmod->2519:sh] do_init_module+0x5c/0x218
[  501.247248] -(3)[2844:insmod->2519:sh] load_module+0x296c/0x31d4
[  501.253283] -(3)[2844:insmod->2519:sh] __se_sys_finit_module+0xd8/0x114
[  501.259928] -(3)[2844:insmod->2519:sh] __arm64_sys_finit_module+0x20/0x2c
[  501.266748] -(3)[2844:insmod->2519:sh] el0_svc_common+0xa0/0x170
[  501.272782] -(3)[2844:insmod->2519:sh] el0_svc_compat_handler+0x28/0x34
[  501.279425] -(3)[2844:insmod->2519:sh] el0_svc_compat+0x8/0x34