watchdog: BUG: soft lockup - CPU#3 stuck for 23s! [swapper/0:1]

测试代码如下：

cpp 复制代码

#include <linux/module.h>
#include <linux/init.h>
#include <linux/kernel.h>

static DEFINE_SPINLOCK(hack_spinA);
static DEFINE_SPINLOCK(hack_spinB);

void hack_spinAB(void)
{
    printk("hack_lockdep:A->B\n");
    spin_lock(&hack_spinA);
    spin_lock(&hack_spinB);
}

void hack_spinBA(void)
{
    printk("hack_lockdep:B->A\n");
    spin_lock(&hack_spinB);
}

static int __init lockdep_test_init(void)
{
    printk("figo:my lockdep module init\n");
    
    hack_spinAB();
    hack_spinBA();
     
    return 0;
}
    
    static void __exit lockdep_test_exit(void)
    {
      printk("goodbye\n");
    }
    
    module_init(lockdep_test_init);
    module_exit(lockdep_test_exit);
    MODULE_LICENSE("GPL");

$28.015539\] -(3)\[1:swapper/0-\>0:swapper/0\]watchdog: BUG: soft lockup - CPU#3 stuck for 23s! \[swapper/0:1$
$28.025300$ -(3) $1:swapper/0-\>0:swapper/0$ Kernel panic - not syncing: softlockup: hung tasks

$28.033776$ -(3) $1:swapper/0-\>0:swapper/0$ CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2

$28.043906$ -(3) $1:swapper/0-\>0:swapper/0$ Hardware name: Platfrom(DT)

$28.050636$ -(3) $1:swapper/0-\>0:swapper/0$ Call trace:

$28.055711$ -(3) $1:swapper/0-\>0:swapper/0$ dump_backtrace+0x0/0x198

$28.062006$ -(3) $1:swapper/0-\>0:swapper/0$ show_stack+0x20/0x2c

$28.067951$ -(3) $1:swapper/0-\>0:swapper/0$ dump_stack+0xd8/0x128

$28.073984$ -(3) $1:swapper/0-\>0:swapper/0$ panic+0x134/0x334

$28.079669$ -(3) $1:swapper/0-\>0:swapper/0$ softlockup_fn+0x0/0x60

$28.085790$ -(3) $1:swapper/0-\>0:swapper/0$ __run_hrtimer+0xa8/0x2b0

$28.092085$ -(3) $1:swapper/0-\>0:swapper/0$ hrtimer_interrupt+0x174/0x3c8

$28.098817$ -(3) $1:swapper/0-\>0:swapper/0$ arch_timer_handler_phys+0x40/0x50

$28.105897$ -(3) $1:swapper/0-\>0:swapper/0$ handle_percpu_devid_irq+0x88/0x278

$28.113065$ -(3) $1:swapper/0-\>0:swapper/0$ __handle_domain_irq+0x84/0xc4

$28.119796$ -(3) $1:swapper/0-\>0:swapper/0$ gic_handle_irq+0x10c/0x184

$28.126265$ -(3) $1:swapper/0-\>0:swapper/0$ el1_irq+0xec/0x198

$28.132035$ -(3) $1:swapper/0-\>0:swapper/0$ queued_spin_lock_slowpath+0x94/0x270

$28.139376$ -(3) $1:swapper/0-\>0:swapper/0$ _raw_spin_lock+0x50/0x54

$28.145670$ -(3) $1:swapper/0-\>0:swapper/0$ lockdep_test_init+0x34/0x40

$28.152226$ -(3) $1:swapper/0-\>0:swapper/0$ do_one_initcall+0xa0/0x298

$28.158694$ -(3) $1:swapper/0-\>0:swapper/0$ do_initcall_level+0xc0/0x164

$28.165337$ -(3) $1:swapper/0-\>0:swapper/0$ do_basic_setup+0x34/0x4c

$28.171631$ -(3) $1:swapper/0-\>0:swapper/0$ kernel_init_freeable+0xd0/0x150

$28.178535$ -(3) $1:swapper/0-\>0:swapper/0$ kernel_init+0x18/0x29c

$28.184655$ -(3) $1:swapper/0-\>0:swapper/0$ ret_from_fork+0x10/0x18

$28.190864$ -(3) $1:swapper/0-\>0:swapper/0$ SMP: stopping secondary CPUs

$28.197425$ -(3) $1:swapper/0-\>0:swapper/0$ Kernel Offset: disabled

$28.203545$ -(3) $1:swapper/0-\>0:swapper/0$ CPU features: 0x00000000,22002a38

$28.210537$ -(3) $1:swapper/0-\>0:swapper/0$ Memory Limit: none

$28.216221$ -(3) $1:swapper/0-\>0:swapper/0$ eskdump die

$28.221295$ -(3) $1:swapper/0-\>0:swapper/0$ CPU: 3 PID: 1 Comm: swapper/0 Tainted: G L 4.19.176 #2

$28.231425$ -(3) $1:swapper/0-\>0:swapper/0$ Hardware name: ES (DT)

$28.238156$ -(3) $1:swapper/0-\>0:swapper/0$ pstate: 60000089 (nZCv daIf -PAN -UAO)

$28.245584$ -(3) $1:swapper/0-\>0:swapper/0$ pc : panic+0x1b8/0x334

$28.251617$ -(3) $1:swapper/0-\>0:swapper/0$ lr : panic+0x148/0x334

$28.257649$ -(3) $1:swapper/0-\>0:swapper/0$ sp : ffff00000801bbe0

$28.263592$ -(3) $1:swapper/0-\>0:swapper/0$ x29: ffff00000801bd80 x28: 0000000000000001

$28.271543$ -(3) $1:swapper/0-\>0:swapper/0$ x27: ffff80037ff9d180 x26: 0000000685682d4d

$28.279494$ -(3) $1:swapper/0-\>0:swapper/0$ x25: ffff00000a4fd000 x24: 0000000000000003

$28.287445$ -(3) $1:swapper/0-\>0:swapper/0$ x23: ffff00000a0208b0 x22: 0000000000000000

$28.295396$ -(3) $1:swapper/0-\>0:swapper/0$ x21: ffff800379219ec0 x20: 0000000000000000

$28.303346$ -(3) $1:swapper/0-\>0:swapper/0$ x19: ffff000009af821d x18: 0000000000000001

$28.311297$ -(3) $1:swapper/0-\>0:swapper/0$ x17: 00000000000cc70c x16: 00000000000000f4

$28.319248$ -(3) $1:swapper/0-\>0:swapper/0$ x15: ffff00000923f160 x14: ffff00000980e250

$28.327199$ -(3) $1:swapper/0-\>0:swapper/0$ x13: 000000000000369a x12: 0000000029aaaaab

$28.335150$ -(3) $1:swapper/0-\>0:swapper/0$ x11: 0000000000000001 x10: 0000000006666998

$28.343100$ -(3) $1:swapper/0-\>0:swapper/0$ x9 : 4b602900c22d1900 x8 : ffff00000801bc20

$28.351051$ -(3) $1:swapper/0-\>0:swapper/0$ x7 : 73203a504d535d30 x6 : ffff00000a8b5b16

$28.359002$ -(3) $1:swapper/0-\>0:swapper/0$ x5 : 0000000000000001 x4 : 000000000000000c

$28.366953$ -(3) $1:swapper/0-\>0:swapper/0$ x3 : 000000000000004a x2 : 0000000000000003

$28.374903$ -(3) $1:swapper/0-\>0:swapper/0$ x1 : 0000000000000004 x0 : 0000000000000001

$28.382854$ -(3) $1:swapper/0-\>0:swapper/0$ Modules linked in:

$28.397593$ -(3) $1:swapper/0-\>0:swapper/0$ done available data size 0x83000

$28.404503$ -(3) $1:swapper/0-\>0:swapper/0$ kdump_sign sig 0x8a6acdb4, size 0x83000

$28.412017$ -(3) $1:swapper/0-\>0:swapper/0$ rebooting...
这里没有在kernel 添加lockdep配置，打印的calltrace信息，测试代码逻辑比较简单，所以能一眼看出来lockdep_test_init中有问题

Linux内核中打开lockdep功能，需要打开CONFIG_DEBUG_LOCKDEP选项：

cpp 复制代码

CONFIG_LOCK_STAT=y
CONFIG_PROVE_LOCKING=y
CONFIG_DEBUG_LOCKDEP=y

cpp 复制代码

console:/ # insmod ./lockdep_test.ko
[  501.016783] -(3)[2844:insmod->2519:sh]
[  501.020560] -(3)[2844:insmod->2519:sh]============================================
[  501.028164] -(3)[2844:insmod->2519:sh]WARNING: possible recursive locking detected
[  501.035768] -(3)[2844:insmod->2519:sh]4.19.176 #1 Tainted: G           O
[  501.042935] -(3)[2844:insmod->2519:sh]--------------------------------------------
[  501.050538] -(3)[2844:insmod->2519:sh]insmod/2844 is trying to acquire lock:
[  501.057617] 00000000f287eb44 (hack_spinB){+.+.}, at: init_module+0x34/0xfb0 [lockdep_test]
[  501.065931] -(3)[2844:insmod->2519:sh]
[  501.065931] but task is already holding lock:
[  501.074056] 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[  501.082275] -(3)[2844:insmod->2519:sh]
[  501.082275] other info that might help us debug this:
[  501.091100] -(3)[2844:insmod->2519:sh] Possible unsafe locking scenario:
[  501.091100]
[  501.099313] -(3)[2844:insmod->2519:sh]       CPU0
[  501.104038] -(3)[2844:insmod->2519:sh]       ----
[  501.108761] -(3)[2844:insmod->2519:sh]  lock(hack_spinB);
[  501.114184] -(3)[2844:insmod->2519:sh]  lock(hack_spinB);
[  501.119608] -(3)[2844:insmod->2519:sh]
[  501.119608]  *** DEADLOCK ***
[  501.119608]
[  501.127823] -(3)[2844:insmod->2519:sh] May be due to missing lock nesting notation
[  501.127823]
[  501.136908] -(3)[2844:insmod->2519:sh]2 locks held by insmod/2844:
[  501.143114] -(3)[2844:insmod->2519:sh] #0: 000000000721ea50 (hack_spinA){+.+.}, at: hack_spinAB+0x24/0x38 [lockdep_test]
[  501.154035] -(3)[2844:insmod->2519:sh] #1: 00000000f287eb44 (hack_spinB){+.+.}, at: hack_spinAB+0x30/0x38 [lockdep_test]
[  501.164958] -(3)[2844:insmod->2519:sh]
[  501.164958] stack backtrace:
[  501.171605] -(3)[2844:insmod->2519:sh]CPU: 3 PID: 2844 Comm: insmod Tainted: G           O      4.19.176 #1
[  501.181388] -(3)[2844:insmod->2519:sh]Hardware name: ESWIN_LD60 (DT)
[  501.187770] -(3)[2844:insmod->2519:sh]Call trace:
[  501.192500] -(3)[2844:insmod->2519:sh] dump_backtrace+0x0/0x198
[  501.198445] -(3)[2844:insmod->2519:sh] show_stack+0x20/0x2c
[  501.204045] -(3)[2844:insmod->2519:sh] dump_stack+0xd8/0x128
[  501.209732] -(3)[2844:insmod->2519:sh] __lock_acquire+0x17d8/0x1ed0
[  501.216028] -(3)[2844:insmod->2519:sh] lock_acquire+0x12c/0x1f4
[  501.221976] -(3)[2844:insmod->2519:sh] _raw_spin_lock+0x54/0x90
[  501.227922] -(3)[2844:insmod->2519:sh] init_module+0x34/0xfb0 [lockdep_test]
[  501.235003] -(3)[2844:insmod->2519:sh] do_one_initcall+0x13c/0x3e0
[  501.241213] -(3)[2844:insmod->2519:sh] do_init_module+0x5c/0x218
[  501.247248] -(3)[2844:insmod->2519:sh] load_module+0x296c/0x31d4
[  501.253283] -(3)[2844:insmod->2519:sh] __se_sys_finit_module+0xd8/0x114
[  501.259928] -(3)[2844:insmod->2519:sh] __arm64_sys_finit_module+0x20/0x2c
[  501.266748] -(3)[2844:insmod->2519:sh] el0_svc_common+0xa0/0x170
[  501.272782] -(3)[2844:insmod->2519:sh] el0_svc_compat_handler+0x28/0x34
[  501.279425] -(3)[2844:insmod->2519:sh] el0_svc_compat+0x8/0x34