1、static key是什么
Linux内核的 Static Keys机制是一种高效的条件分支优化技术,主要用于在运行时动态启用或禁用特定代码路径,同时避免常规条件判断(如 if 语句)的性能开销。它通过结合编译时优化和运行时代码修补(如 Jump Label 技术)实现近乎零成本的开关切换,广泛应用于性能敏感的代码逻辑(如调试代码、跟踪点、性能监控等)。
1.1、核心原理
- 无分支执行:Static Keys将条件判断转换为对内存中某个标志位的直接跳转,绕过传统if语句的分支预测和流水线冲刷。
- 代码修补:通过修改内存中的指令(如将 nop 替换为 jmp),在运行时动态启用或禁用代码路径(依赖CPU架构支持，如x86的jmp指令替换)。
- 原子性操作:启用或禁用Static Key是原子操作，无需锁机制，保证线程安全。
1.2、核心数据结构
1.2.1、struct static_key
表示一个静态键的状态,关键字段如下：
struct static_key {
atomic_t enabled;
/* Jump Label 相关字段（如跳转地址） */
};enabled:键的启用状态(0 禁用，1 启用)。
1.2.2、键的初始状态
DEFINE_STATIC_KEY_TRUE(key): 定义初始为true的键。
DEFINE_STATIC_KEY_FALSE(key): 定义初始为false的键。
1.2.3、使用方式
(1)、定义静态键
#include <linux/jump_label.h>
/*定义并初始化一个静态键(默认禁用)*/
DEFINE_STATIC_KEY_FALSE(my_key);
(2)、在代码中使用静态键
if (static_branch_unlikely(&my_key)) {
/*仅当 my_key 启用时执行此代码块*/
pr_info("Feature is enabled!\n");
}
- 分支预测提示：
static_branch_unlikely():暗示代码块大概率不执行(优化跳转指令位置)。
static_branch_likely():暗示代码块大概率执行。
(3)、动态启用/禁用键
/*启用键*/
static_branch_enable(&my_key);
/*禁用键*/
static_branch_disable(&my_key);
2、数据结构
typedef u32 jump_label_t;
struct jump_entry {
jump_label_t code; /*被内核动态修改的nop或跳转指令地址 */
jump_label_t target; /*l_yes 标号地址，arch_static_branch()函数中会描述 l_yes 标号*/
/*
关联的static_key的地址，static_key的地址4字节对齐,地址值最低2bit位总是0。
内核实现利用最低1位用来标识 branch 类型:
0 -> false (static_key_false()构建的 jump_entry )
1 -> true (static_key_true()构建的 jump_entry )
*/
jump_label_t key;
};
struct static_key_mod {
struct static_key_mod *next;
struct jump_entry *entries;
struct module *mod;
};
struct static_key {
atomic_t enabled;
/*
* Note:
* To make anonymous unions(匿名联合) work with old compilers, the static
* initialization of them requires brackets(大括号). This creates a dependency
* on the order of the struct with the initializers. If any fields
* are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need
* to be modified.
*
* bit 0 => 1 if key is initially true
* 0 if initially false
* bit 1 => 1 if points to struct static_key_mod
* 0 if points to struct jump_entry
* 因为地址是4字节对齐,所以可以将低2bit作为标记使用
*/
union {
unsigned long type;
struct jump_entry *entries;
struct static_key_mod *next;
};
};
/*
* Two type wrappers around static_key, such that we can use compile time type differentiation to emit the right code.
* All the below code is macros in order to play type games.
*/
struct static_key_true {
struct static_key key;
};
struct static_key_false {
struct static_key key;
};
enum jump_label_type {
JUMP_LABEL_NOP = 0,
JUMP_LABEL_JMP,
};
#define JUMP_TYPE_FALSE 0UL
#define JUMP_TYPE_TRUE 1UL
#define JUMP_TYPE_LINKED 2UL
#define JUMP_TYPE_MASK 3UL
下面是static key声明和定义
/*
* We should be using ATOMIC_INIT() for initializing .enabled, but
* the inclusion of atomic.h is problematic for inclusion of jump_label.h
* in 'low-level' headers. Thus, we are initializing .enabled with a
* raw value, but have added a BUILD_BUG_ON() to catch any issues in
* jump_label_init() see: kernel/jump_label.c.
*/
/*enable为1，entries为1*/
#define STATIC_KEY_INIT_TRUE \
{ .enabled = { 1 }, \
{ .entries = (void *)JUMP_TYPE_TRUE } }
/*enable为0,entries为0*/
#define STATIC_KEY_INIT_FALSE \
{ .enabled = { 0 }, \
{ .entries = (void *)JUMP_TYPE_FALSE } }
/*根据类型(true/false)来初始化.key成员变量*/
#define STATIC_KEY_TRUE_INIT (struct static_key_true) { .key = STATIC_KEY_INIT_TRUE, }
#define STATIC_KEY_FALSE_INIT (struct static_key_false){ .key = STATIC_KEY_INIT_FALSE, }
#define DEFINE_STATIC_KEY_TRUE(name) \
struct static_key_true name = STATIC_KEY_TRUE_INIT
#define DECLARE_STATIC_KEY_TRUE(name) \
extern struct static_key_true name
#define DEFINE_STATIC_KEY_FALSE(name) \
struct static_key_false name = STATIC_KEY_FALSE_INIT
#define DECLARE_STATIC_KEY_FALSE(name) \
extern struct static_key_false name
#define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count) \
struct static_key_true name[count] = { \
[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT, \
}
#define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count) \
struct static_key_false name[count] = { \
[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT, \
}
enum jump_label_type {
JUMP_LABEL_NOP = 0,
JUMP_LABEL_JMP,
};3、接口分析
static_branch_likely()
static_branch_unlikely()
static_branch_enable()
static_branch_disable()
static_branch_inc()
static_branch_dec()
3.1、static_branch_likely/static_branch_unlikely()分析
/*
* Combine the right initial value (type) with the right branch order to generate the desired result.
* 注意 likely(...)优化后会将成功放在前面,失败放在后面；unlikely(...)优化后会将失败放在前面,成功放在后面；
* type为true表示struct static_key_true，type为false表示struct static_key_false
* type\branch| likely (1) | unlikely (0)
* -----------+-----------------------+------------------
* | |
* true (1) | ... | ...
* | NOP | JMP L
* | <br-stmts> | 1: ...
* | L: ... |
* | |
* | | L: <br-stmts>
* | | jmp 1b
* | |
* -----------+-----------------------+------------------
* | |
* false (0) | ... | ...
* | JMP L | NOP
* | <br-stmts> | 1: ...
* | L: ... |
* | |
* | | L: <br-stmts>
* | | jmp 1b
* | |
* -----------+-----------------------+------------------
*
* The initial value is encoded in the LSB of static_key::entries,
* type: 0 = false, 1 = true.
*
* The branch type is encoded in the LSB of jump_entry::key,
* branch: 0 = unlikely, 1 = likely.
*
* This gives the following logic table:
*
* enabled type branch instuction
* -----------------------------+-----------
* 0 0 0 | NOP
* 0 0 1 | JMP
* 0 1 0 | NOP
* 0 1 1 | JMP
*
* 1 0 0 | JMP
* 1 0 1 | NOP
* 1 1 0 | JMP
* 1 1 1 | NOP
*
* Which gives the following functions:
*
* dynamic: instruction = enabled ^ branch (动态时取决于enable和branch的值)
* static: instruction = type ^ branch (静态时取决于type和branch的值)
* 上面这个规则很重要
* See jump_label_type() / jump_label_init_type().
*/在Jump Label机制中,type和branch是两个关键参数,用于控制动态分支的行为和优化方向。
branch是一个布尔值(true/false 或 1/0),表示当前分支的默认状态(是否启用),它直接影响生成的指令类型(如nop或jmp),以及static_key的地址偏移计算。
决定指令初始值：
branch=true,初始指令为jmp(分支启用)。
branch=false,初始指令为nop(分支禁用)。
影响static_key的地址偏移:static_key结构体中通过偏移量区分不同的分支状态。
#define static_branch_likely(key) \
arch_static_branch(&(key)->enabled, true)
#define static_branch_unlikely(key) \
arch_static_branch(&(key)->enabled, false)
type是一个枚举值,表示static_key的类型,用于区分不同用途的键,它影响内核对键的初始化和运行时处理。
type和branch协作
初始化阶段
branch决定初始指令:若branch=true,初始指令为 jmp,对应 JUMP_LABEL_JMP。
type 决定处理方式:若type=JUMP_LABEL_TYPE_BRANCH,内核在初始化时会跳过该键,直到显式启用。
运行时修改
启用分支:调用 static_key_enable()将type设为JUMP_LABEL_JMP,并修改指令为jmp。
禁用分支:调用 static_key_disable()将type设为 JUMP_LABEL_NOP,并修改指令为nop。
地址偏移计算
branch 影响 key 的偏移:&((char *)key)[branch] 根据branch的值选择key的地址偏移,确保内核通过static_key的地址快速定位状态。
branch 决定指令的初始状态，type 决定键的生命周期和用途，两者共同实现灵活的动态分支管理。
static key的类型必须为struct static_key_true/struct static_key_false,否则会产生错误。
#define static_branch_likely(x) \
({
/*__builtin_types_compatible_p GNU扩展,用来判断两个类型是否相同,sizeof(int) sizeof(char*)结果一样但是类型不同*/ \
bool branch; \
if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
branch = !arch_static_branch(&(x)->key, true); \
else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
branch = !arch_static_branch_jump(&(x)->key, true); \
else \
branch = ____wrong_branch_error(); \
branch; \
})
注意,static_branch_likely内部调用的arch_static_branch或者arch_static_branch_jump传递的branch都为true;
#define static_branch_unlikely(x) \
({ \
bool branch; \
if (__builtin_types_compatible_p(typeof(*x), struct static_key_true)) \
branch = arch_static_branch_jump(&(x)->key, false); \
else if (__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
branch = arch_static_branch(&(x)->key, false); \
else \
branch = ____wrong_branch_error(); \
branch; \
})
注意,static_branch_unlikely内部调用的arch_static_branch或者arch_static_branch_jump传递的branch都为false;
#define static_key_enabled(x) \
({ \
if (!__builtin_types_compatible_p(typeof(*x), struct static_key) && \
!__builtin_types_compatible_p(typeof(*x), struct static_key_true) &&\
!__builtin_types_compatible_p(typeof(*x), struct static_key_false)) \
____wrong_branch_error(); \
static_key_count((struct static_key *)x) > 0; \
})
int static_key_count(struct static_key *key)
{
/*
* -1 means the first static_key_slow_inc() is in progress.
* static_key_enabled() must return true, so return 1 here.
*/
int n = atomic_read(&key->enabled);
return n >= 0 ? n : 1;
}3.2、arch_static_branch/arch_static_branch_jump 分析
arch_static_branch/arch_static_branch_jump是和cpu体系架构相关的代码,使用asm goto内嵌汇编实现运行时的准确分支功能；
you can reference labels using the actual C label name enclosed in brackets.
For example, to reference a label named carry, you can use '%l[carry]'.
The label must still be listed in the GotoLabels section when using this approach.
#define JUMP_LABEL_NOP_SIZE 4
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
{
asm_volatile_goto("1:\n\t"
/*运行时被动态修改的指令或修改为nop,或修改为跳转到 l_yes 标号的指令*/
/*此时为nop空操作指令*/
WASM(nop) "\n\t"
/*
在 __jump_table的section内,构建 jump_entry 变量：
(struct jump_entry){
.code = 上面标号1的地址,即运行是被动态修改指令的地址;
.target = 标号 l_yes 地址;
.key = key 地址,用最低1位标记branch类型: false(0), true(1),(char *)key将key地址转换为char*可以进行字节偏移
&((char *)key)[branch])通过key &~1可以恢复static_key的地址,同时key地址最低bit位为branch的值(likely为true,unlikely为false);
因为static key的地址是4字节对齐的,所以可以使用上面的方法,非常巧妙的方法。
}
*/
".pushsection __jump_table, \"aw\"\n\t"
".word 1b, %l[l_yes], %c0\n\t"
".popsection\n\t"
: : "i" (&((char *)key)[branch]) : : l_yes);
return false;
l_yes:
return true;
}
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
{
asm_volatile_goto("1:\n\t"
/*运行时被动态修改的指令或修改为nop,或修改为跳转到 l_yes 标号的指令*/
/*此时为跳转指令 b l_yes*/
WASM(b) " %l[l_yes]\n\t"
/*
在__jump_table的section内,构建 jump_entry 变量：
(struct jump_entry){
.code = 上面标号1的地址,即运行是被动态修改指令的地址;
.target = 标号 l_yes 地址;
.key = key 地址,用最低1位标记branch类型: false(0), true(1),(char *)key将key地址转换为char*可以进行字节偏移
&((char *)key)[branch])通过key &~1可以恢复static_key的地址,同时key地址最低bit位为branch的值(likely为true,unlikely为false);
因为static key的地址是4字节对齐的,所以可以使用上面的方法,非常巧妙的方法。
}
*/
".pushsection __jump_table, \"aw\"\n\t" /*切换到__jump_table段*/
".word 1b, %l[l_yes], %c0\n\t" /*往段中写入jump_entry 数据*/
".popsection\n\t"
: : "i" (&((char *)key)[branch]) : : l_yes); /*切回.text段*/
return false;
l_yes:
return true;
}上面代码中的".word 1b, %l[l_yes], %c0\n\t"
.word 定义字空间,会依次将1b,%l[l_yes],%c0 数据放到从当前位置开始的连续地址空间上,即__jump_table段中。
跳转表与代码修补：__jump_table 段记录所有可动态修改的跳转点。
内核运行时(如调用 static_branch_enable())遍历此表，修改代码指令。
在内联汇编中,%c0是GCC的操作数修饰符,用于将输入操作数强制作为立即数(Immediate Value)嵌入到汇编指令中。
上面的Jump Label的代码中用于将static_key的地址偏移直接写入跳转表条目。
语法含义
%c0：
%0:表示第一个输入操作数(索引从 0 开始)。
c修饰符,强制将操作数视为立即数(即直接嵌入指令中的常量值)。
约束条件 "i":在输入操作数约束中,"i"表示操作数必须是整数类型的编译时常量(如 #define 定义的常量或地址偏移)。
注意:
当arch_static_branch/arch_static_branch_jump()函数被编译时,内嵌汇编中1:标签处的内容为nop指令或者b指令在内核的代码段中,内核加载时,其被加载到内存中的某个地址上。
而pushsection和popsection指令在编译时已经将定义的struct jump_entry数据(".word 1b, %l[l_yes], %c0\n\t")存入__jump_table段中,所以在后面要介绍的
jump_label_init/jump_label_update()函数将标签1:(jump_entry->code)地址中的指令修改为nop或者b指令,从而达到运行时动态启用或禁用特定代码路径的功能。
当arch_static_branch/arch_static_branch_jump()被调用时,会根据key的状态,决定是否跳转到l_yes标签。
如果跳转,返回true,否则返回false。和if判断条件相比,不用每次调用都去进行一次条件判断,提高执行效率。
这里的关键在于内联汇编如何与跳转表配合,让内核在运行时通过修改代码(如将nop替换为jmp)来改变执行路径。
static __always_inline bool static_key_false(struct static_key *key)
{
return arch_static_branch(key, false);
}
static __always_inline bool static_key_true(struct static_key *key)
{
return !arch_static_branch(key, true);
}对于函数对 static_key_*() 前的 __always_inline 修饰符，还得额外说明一下， __always_inline 是告诉编译器，函数代码必须就地展开。
在当前的场景下，不加__always_inline 修饰行不行？答案是不行。因为函数 arch_static_branch() 里面使用了标号 1 和 标号 l_yes 的地址，
为了保证每个 struct jump_entry 记录的标号地址唯一，必须要就地展开函数代码。
3.3、jump_label_init 分析
经过前面的分析可知,所有的struct jump_entry都放在名为__jump_table section内,在链接阶段,内核链接脚本vmlinux.lds片段
...
. = ALIGN(8); __start___jump_table = .; *(__jump_table) __stop___jump_table = .; . = ALIGN(8);
...
将所有的 __jump_table输入section,放置到__start___jump_table到 __stop___jump_table区间内,且保证每个 struct jump_entry 的地址都位于4字节边界;
ALIGN(8) 保证了第一个struct jump_entry的地址位于8字节边界，同时每个struct jump_entry 变量为 4 * 3 = 12 字节。
地址4字节对齐，意味着地址的低2位总是为0，正是利用这一点，struct static_key::type用低位的2位来存储相关struct jump_entry(即static_key::entries指向的struct jump_entry)的类型。
arch/arm/kernel/jump_label.c
start_kernel->jump_label_init
void __init jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
struct static_key *key = NULL;
struct jump_entry *iter;
/*
* Since we are initializing the static_key.enabled field with
* with the 'raw' int values (to avoid pulling in atomic.h) in
* jump_label.h, let's make sure that is safe. There are only two
* cases to check since we initialize to 0 or 1.
*/
BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);
if (static_key_initialized)
return;
cpus_read_lock();
jump_label_lock();
/*通过jump_entry key值来排序*/
jump_label_sort_entries(iter_start, iter_stop);
/*遍历所有的jump_entry*/
for (iter = iter_start; iter < iter_stop; iter++) {
struct static_key *iterk;
/* rewrite NOPs */
if (jump_label_type(iter) == JUMP_LABEL_NOP) /*nop类型,将jump_entry->code位置替换为nop指令*/
arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
/*获取jump_entry对应的static_key*/
iterk = jump_entry_key(iter);
if (iterk == key)
continue;
key = iterk;
/*设置static_key的entries和type值,将static_key和jump_entry建立联系*/
static_key_set_entries(key, iter);
}
/*static_key初始化完成*/
static_key_initialized = true;
jump_label_unlock();
cpus_read_unlock();
}
/*获取jump_entry对应的static_key地址*/
static inline struct static_key *jump_entry_key(struct jump_entry *entry)
{
/*
在arch_static_branch/arch_static_branch_jump中设置的key值为struct static_key地址偏移一个地址(即(&((char *)key)[branch])，
这里获取struct jump_entry 对应的struct static_key结构。
*/
return (struct static_key *)((unsigned long)entry->key & ~1UL);
}
static bool jump_entry_branch(struct jump_entry *entry)
{
/*通过地址判断brach为0或为1,key的值设置为(&((char *)key)[branch]。*/
return (unsigned long)entry->key & 1UL;
}
static enum jump_label_type jump_label_type(struct jump_entry *entry)
{
/*获取jump_entry对应的static_key地址*/
struct static_key *key = jump_entry_key(entry);
/*static_key的enabled是否>0,即是否使能当前static key*/
bool enabled = static_key_enabled(key);
/*通过地址判断brach为0或为1*/
bool branch = jump_entry_branch(entry);
/* See the comment in linux/jump_label.h */
/*计算出当前放置的是not指令还是jump指令*/
return enabled ^ branch;
}
/***
* A 'struct static_key' uses a union such that it either points directly
* to a table of 'struct jump_entry' or to a linked list of modules which in
* turn point to 'struct jump_entry' tables.
*
* The two lower bits of the pointer are used to keep track of which pointer
* type is in use and to store the initial branch direction, we use an access
* function which preserves these bits.
*/
static void static_key_set_entries(struct static_key *key,struct jump_entry *entries)
{
unsigned long type;
WARN_ON_ONCE((unsigned long)entries & JUMP_TYPE_MASK);
/*在对entries赋值之前将key->type值记录,记录其初始值,这里使用了地址对齐的小技巧来存放JUMP_TYPE_FALSE/JUMP_TYPE_TRUE*/
type = key->type & JUMP_TYPE_MASK;
/*将static_key和jump_entry建立联系*/
key->entries = entries;
/*将type值写回*/
key->type |= type;
}
void arch_jump_label_transform_static(struct jump_entry *entry,enum jump_label_type type)
{
__arch_jump_label_transform(entry, type, true);
}
static void __arch_jump_label_transform(struct jump_entry *entry,enum jump_label_type type,bool is_static)
{
void *addr = (void *)entry->code;
unsigned int insn;
if (type == JUMP_LABEL_JMP) /*产生跳转指令*/
insn = arm_gen_branch(entry->code, entry->target);
else
insn = arm_gen_nop(); /*产生nop指令*/
/*修改addr处的指令为insn*/
if (is_static)
__patch_text_early(addr, insn);
else
patch_text(addr, insn);
}
static inline unsigned long arm_gen_nop(void)
{
#ifdef CONFIG_THUMB2_KERNEL
return 0xf3af8000; /* nop.w */
#else
return 0xe1a00000; /* mov r0, r0 */
#endif
}
static inline unsigned long arm_gen_branch(unsigned long pc, unsigned long addr)
{
return __arm_gen_branch(pc, addr, false);
}
static inline unsigned long arm_gen_branch_link(unsigned long pc, unsigned long addr)
{
return __arm_gen_branch(pc, addr, true);
}
unsigned long __arm_gen_branch(unsigned long pc, unsigned long addr, bool link)
{
if (IS_ENABLED(CONFIG_THUMB2_KERNEL))
return __arm_gen_branch_thumb2(pc, addr, link);
else
return __arm_gen_branch_arm(pc, addr, link);
}
static unsigned long __arm_gen_branch_arm(unsigned long pc, unsigned long addr, bool link)
{
unsigned long opcode = 0xea000000;
long offset;
if (link) /*link寄存器*/
opcode |= 1 << 24;
offset = (long)addr - (long)(pc + 8);
if (unlikely(offset < -33554432 || offset > 33554428)) {
WARN_ON_ONCE(1);
return 0;
}
offset = (offset >> 2) & 0x00ffffff;
return opcode | offset;
}
/*静态修改内存地址对应的指令*/
static inline void __patch_text_early(void *addr, unsigned int insn)
{
__patch_text_real(addr, insn, false);
}
/*动态修改内存地址对应的指令*/
static inline void __patch_text(void *addr, unsigned int insn)
{
__patch_text_real(addr, insn, true);
}
void __kprobes __patch_text_real(void *addr, unsigned int insn, bool remap)
{
bool thumb2 = IS_ENABLED(CONFIG_THUMB2_KERNEL);
unsigned int uintaddr = (uintptr_t) addr;
bool twopage = false;
unsigned long flags;
void *waddr = addr;
int size;
if (remap)
waddr = patch_map(addr, FIX_TEXT_POKE0, &flags);
else
__acquire(&patch_lock);
if (thumb2 && __opcode_is_thumb16(insn)) {
*(u16 *)waddr = __opcode_to_mem_thumb16(insn);
size = sizeof(u16);
} else if (thumb2 && (uintaddr & 2)) {
u16 first = __opcode_thumb32_first(insn);
u16 second = __opcode_thumb32_second(insn);
u16 *addrh0 = waddr;
u16 *addrh1 = waddr + 2;
twopage = (uintaddr & ~PAGE_MASK) == PAGE_SIZE - 2;
if (twopage && remap)
addrh1 = patch_map(addr + 2, FIX_TEXT_POKE1, NULL);
*addrh0 = __opcode_to_mem_thumb16(first);
*addrh1 = __opcode_to_mem_thumb16(second);
if (twopage && addrh1 != addr + 2) {
flush_kernel_vmap_range(addrh1, 2);
patch_unmap(FIX_TEXT_POKE1, NULL);
}
size = sizeof(u32);
} else {
if (thumb2)
insn = __opcode_to_mem_thumb32(insn);
else
insn = __opcode_to_mem_arm(insn);
*(u32 *)waddr = insn;
size = sizeof(u32);
}
if (waddr != addr) {
flush_kernel_vmap_range(waddr, twopage ? size / 2 : size);
patch_unmap(FIX_TEXT_POKE0, &flags);
} else
__release(&patch_lock);
/*冲刷指令cache,避免cache不一致*/
flush_icache_range((uintptr_t)(addr),
(uintptr_t)(addr) + size);
}
实时动态更新jump_label_update
/*更新对应的static_key*/
static void jump_label_update(struct static_key *key)
{
struct jump_entry *stop = __stop___jump_table;
struct jump_entry *entry;
#ifdef CONFIG_MODULES
struct module *mod;
if (static_key_linked(key)) {
__jump_label_mod_update(key);
return;
}
preempt_disable();
mod = __module_address((unsigned long)key);
if (mod)
stop = mod->jump_entries + mod->num_jump_entries;
preempt_enable();
#endif
/*static_key对应的jump_entry*/
entry = static_key_entries(key);
/* if there are no users, entry can be NULL */
if (entry)
__jump_label_update(key, entry, stop);
}
static void __jump_label_update(struct static_key *key,struct jump_entry *entry,struct jump_entry *stop)
{
for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
/*
* entry->code set to 0 invalidates module init text sections
* kernel_text_address() verifies we are not in core kernel
* init code, see jump_label_invalidate_module_init().
*/
if (entry->code && kernel_text_address(entry->code)) /*是否为kernel代码段地址*/
arch_jump_label_transform(entry, jump_label_type(entry));
}
}3.4、动态使能/失能static key
/*
* Advanced usage; refcount, branch is enabled when: count != 0
*/
#define static_branch_inc(x) static_key_slow_inc(&(x)->key)
#define static_branch_dec(x) static_key_slow_dec(&(x)->key)
#define static_branch_inc_cpuslocked(x) static_key_slow_inc_cpuslocked(&(x)->key)
#define static_branch_dec_cpuslocked(x) static_key_slow_dec_cpuslocked(&(x)->key)
/*
* Normal usage; boolean enable/disable.
*/
#define static_branch_enable(x) static_key_enable(&(x)->key)
#define static_branch_disable(x) static_key_disable(&(x)->key)
#define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key)
#define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key)
static_key_enable流程分析如下:
void static_key_enable(struct static_key *key)
{
cpus_read_lock();
static_key_enable_cpuslocked(key);
cpus_read_unlock();
}
void static_key_enable_cpuslocked(struct static_key *key)
{
STATIC_KEY_CHECK_USE();
/*获取key的enable值,>0则warn,理论上其值应为0*/
if (atomic_read(&key->enabled) > 0) {
WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
return;
}
jump_label_lock();
if (atomic_read(&key->enabled) == 0) {
/*将enabled设置为-1,update过程中其值为-1*/
atomic_set(&key->enabled, -1);
/*更新jump_label*/
jump_label_update(key);
/*
* See static_key_slow_inc().
*/
/*设置为1*/
atomic_set_release(&key->enabled, 1);
}
jump_label_unlock();
}
/*更新对应的static_key*/
static void jump_label_update(struct static_key *key)
{
struct jump_entry *stop = __stop___jump_table;
struct jump_entry *entry;
#ifdef CONFIG_MODULES
struct module *mod;
if (static_key_linked(key)) {
__jump_label_mod_update(key);
return;
}
preempt_disable();
mod = __module_address((unsigned long)key);
if (mod)
stop = mod->jump_entries + mod->num_jump_entries;
preempt_enable();
#endif
/*static_key对应的jump_entry*/
entry = static_key_entries(key);
/* if there are no users, entry can be NULL */
if (entry)
__jump_label_update(key, entry, stop);
}
static void __jump_label_update(struct static_key *key,struct jump_entry *entry,struct jump_entry *stop)
{
for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
/*
* entry->code set to 0 invalidates module init text sections
* kernel_text_address() verifies we are not in core kernel
* init code, see jump_label_invalidate_module_init().
*/
if (entry->code && kernel_text_address(entry->code)) /*是否为kernel代码段地址*/
arch_jump_label_transform(entry, jump_label_type(entry)); /*由于enabled的值发生变化jump_label_type值也会变换,根据type类型决定是nop还是b跳转指令*/
}
}
static_key_disable实现
void static_key_disable(struct static_key *key)
{
cpus_read_lock();
static_key_disable_cpuslocked(key);
cpus_read_unlock();
}
void static_key_disable_cpuslocked(struct static_key *key)
{
STATIC_KEY_CHECK_USE();
/*获取key的enable值,!=1则warn,理论上其值应为1*/
if (atomic_read(&key->enabled) != 1) {
WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
return;
}
jump_label_lock();
if (atomic_cmpxchg(&key->enabled, 1, 0)) /*enabled值为1时,原子地修改为0,返回旧值1,否则不做任何操作*/
jump_label_update(key);
jump_label_unlock();
}
static_key_slow_inc实现
void static_key_slow_inc(struct static_key *key)
{
cpus_read_lock();
static_key_slow_inc_cpuslocked(key);
cpus_read_unlock();
}
void static_key_slow_inc_cpuslocked(struct static_key *key)
{
int v, v1;
STATIC_KEY_CHECK_USE();
/*
* Careful if we get concurrent static_key_slow_inc() calls;
* later calls must wait for the first one to _finish_ the
* jump_label_update() process. At the same time, however,
* the jump_label_update() call below wants to see
* static_key_enabled(&key) for jumps to be updated properly.
*
* So give a special meaning to negative key->enabled: it sends
* static_key_slow_inc() down the slow path, and it is non-zero
* so it counts as "enabled" in jump_label_update(). Note that
* atomic_inc_unless_negative() checks >= 0, so roll our own.
*/
for (v = atomic_read(&key->enabled); v > 0; v = v1) { /*enabled>0的情况下只做+1处理*/
v1 = atomic_cmpxchg(&key->enabled, v, v + 1); /*原子加1,v1记录旧值*/
if (likely(v1 == v))
return;
}
jump_label_lock();
if (atomic_read(&key->enabled) == 0) { /*enabled由0->1，需要update*/
atomic_set(&key->enabled, -1);
jump_label_update(key);
/*
* Ensure that if the above cmpxchg loop observes our positive
* value, it must also observe all the text changes.
*/
atomic_set_release(&key->enabled, 1);
} else {
atomic_inc(&key->enabled);
}
jump_label_unlock();
}
void static_key_slow_dec(struct static_key *key)
{
STATIC_KEY_CHECK_USE();
__static_key_slow_dec(key, 0, NULL);
}
static void __static_key_slow_dec(struct static_key *key,unsigned long rate_limit,struct delayed_work *work)
{
cpus_read_lock();
__static_key_slow_dec_cpuslocked(key, rate_limit, work);
cpus_read_unlock();
}
static void __static_key_slow_dec_cpuslocked(struct static_key *key,unsigned long rate_limit,struct delayed_work *work)
{
/*
* The negative count check is valid even when a negative
* key->enabled is in use by static_key_slow_inc(); a
* __static_key_slow_dec() before the first static_key_slow_inc()
* returns is unbalanced, because all other static_key_slow_inc()
* instances block while the update is in progress.
*/
/*enabled原子地-1后值为0,持有jump_label_mutex返回true,否则不持有jump_label_mutex，返回false*/
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
WARN(atomic_read(&key->enabled) < 0,
"jump label: negative count!\n");
return;
}
if (rate_limit) {
atomic_inc(&key->enabled);
schedule_delayed_work(work, rate_limit);
} else {
jump_label_update(key);
}
jump_label_unlock();
}
/**
* atomic_add_unless - add unless the number is already a given value
* @v: pointer of type atomic_t
* @a: the amount to add to v...
* @u: ...unless v is equal to u.
*
* Atomically adds @a to @v, so long as @v was not already @u.
* Returns non-zero if @v was not @u, and zero otherwise.
*/
static inline int atomic_add_unless(atomic_t *v, int a, int u)
{
return __atomic_add_unless(v, a, u) != u;
}
/**
* atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
* @cnt: the atomic which we are to dec
* @lock: the mutex to return holding if we dec to 0
*
* return true and hold lock if we dec to 0, return false otherwise
*/
int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
{
/* dec if we can't possibly hit 0 */
/*除非cnt为1,否则cnt值原子地-1,即cnt不能为0*/
if (atomic_add_unless(cnt, -1, 1)) /*cnt不为1,则原子地-1,返回非0值*/
return 0;
/*下面处理cnt为1的情况*/
/* we might hit 0, so take the lock */
mutex_lock(lock);
/*cnt原子地-1,测试cnt值，如果cnt为0返回true,否则返回false*/
if (!atomic_dec_and_test(cnt)) {
/*cnt原子地-1后其值非0,则释放mutex*/
/* when we actually did the dec, we didn't hit 0 */
mutex_unlock(lock);
return 0;
}
/* we hit 0, and we hold the lock */
return 1;
}