Per-CPU变量是Linux内核中实现高效并发访问的核心机制之一,它为每个处理器维护独立的变量副本,避免了多处理器环境下的锁竞争问题。Per-CPU变量为每个CPU核心创建独立副本的变量,消除多处理器并发访问的竞争条件。本地CPU可直接读写自己的副本,访问其他CPU副本需特殊处理。
在开始讲述Per-CPU内核接口前，根据内核中静态Per-CPU的实现机制(阅读源码)，笔者在用户态手动实现了一个简洁版Per-CPU玩具,其中省略了很多对齐以及其他检查部分，这样可以加深大家的理解，当然希望继续探索的读者可以阅读笔者的文章《linux 内核Per-CPU变量实现分析》,该文章中对内核中涉及Per-CPU的代码进行了详细的注释。
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
/*模拟在".data..percpu"中的Per-CPU变量*/
struct none_t {
unsigned int __per_cpu_start;
int per_a;
int per_b;
int per_c;
long Per_d;
unsigned int __per_cpu_end;
}__attribute__((__aligned__(4)));
/*CPU个数为4*/
#define NR_CPUS 4
unsigned long __per_cpu_offset[NR_CPUS];
unsigned long per_cpu_offset[NR_CPUS];
/*获取per_cpu变量的值*/
#define per_cpu_get_var(var,cpu) \
({ \
*(typeof(var) *)((unsigned long)(&(var)) - per_cpu_offset[cpu]);\
})
/*设置per_cpu变量的值*/
#define per_cpu_set_var(var,cpu,val) \
do{ \
*(typeof(var) *)((unsigned long)(&(var)) - per_cpu_offset[cpu]) = val; \
}while(0)
int main()
{
int i=0,cpu=0;
size_t size = 0;
struct none_t n;
void * per_cpu_areas = NULL;
unsigned long delta=0;
unsigned long *offsets;
printf("%ld \n",sizeof(offsets[0])); //8
/*4,8,8,4,8 */
printf("%zu,%zu,%zu,%zu,%zu \n",sizeof(int),sizeof(long),sizeof(long long),sizeof(float),sizeof(double));
printf("__per_cpu_start:%p,per_a:%p,per_b:%p,per_c:%p,Per_d:%p,__per_cpu_end:%p \n",\
&n.__per_cpu_start,&n.per_a,&n.per_b,&n.per_c,&n.Per_d,&n.__per_cpu_end);
/*为静态Per-CPU变量分配空间*/
size = sizeof(struct none_t);
per_cpu_areas = malloc(NR_CPUS * size);
printf("per_cpu_areas=%p,size=%ld,0x%zu \n",per_cpu_areas,size,sizeof(struct none_t));
if(!per_cpu_areas) {
printf("malloc memory failure. \n");
return -1;
}
for(i=0; i<NR_CPUS; i++) {
__per_cpu_offset[i] = i * size;
printf("__per_cpu_offset[%d]:0x%lx \n",i,__per_cpu_offset[i]);
}
/*拷贝Per-CPU数据到每个cpu对应的内存空间中*/
for(i=0; i<NR_CPUS; i++) {
memcpy((void *)((unsigned long)per_cpu_areas+__per_cpu_offset[i]),(void *)(unsigned long)&n.__per_cpu_start,size);
}
delta = (unsigned long)(&n.__per_cpu_start)-(unsigned long)per_cpu_areas;
printf("delta:0x%lx,per_cpu_areas:0x%lx \n",delta,(unsigned long)per_cpu_areas);
for(i=0; i<NR_CPUS; i++) {
per_cpu_offset[i] = __per_cpu_offset[i] + delta;
printf("per_cpu_offset[%d]:0x%lx \n",i,per_cpu_offset[i]);
}
per_cpu_set_var(n.per_a,0,0x10);
per_cpu_set_var(n.per_a,1,0x11);
per_cpu_set_var(n.per_a,2,0x12);
per_cpu_set_var(n.per_a,3,0x13);
printf("0x%x \n",per_cpu_get_var(n.per_a,0));
printf("0x%x \n",per_cpu_get_var(n.per_a,1));
printf("0x%x \n",per_cpu_get_var(n.per_a,2));
printf("0x%x \n",per_cpu_get_var(n.per_a,3));
per_cpu_set_var(n.Per_d,0,0xf10);
per_cpu_set_var(n.Per_d,1,0xf11);
per_cpu_set_var(n.Per_d,2,0xf12);
per_cpu_set_var(n.Per_d,3,0xf13);
printf("0x%lx \n",per_cpu_get_var(n.Per_d,0));
printf("0x%lx \n",per_cpu_get_var(n.Per_d,1));
printf("0x%lx \n",per_cpu_get_var(n.Per_d,2));
printf("0x%lx \n",per_cpu_get_var(n.Per_d,3));
if(per_cpu_areas)
free(per_cpu_areas);
return 0;
}/*
打印结果如下:
8
4,8,8,4,8
__per_cpu_start:0x7ffe8af08950,per_a:0x7ffe8af08954,per_b:0x7ffe8af08958,per_c:0x7ffe8af0895c,Per_d:0x7ffe8af08960,__per_cpu_end:0x7ffe8af08968
per_cpu_areas=0x1ea5420,size=32,0x32
__per_cpu_offset[0]:0x0
__per_cpu_offset[1]:0x20
__per_cpu_offset[2]:0x40
__per_cpu_offset[3]:0x60
delta:0x89063530,per_cpu_areas:0x1ea5420
per_cpu_offset[0]:0x7ffe89063530
per_cpu_offset[1]:0x7ffe89063550
per_cpu_offset[2]:0x7ffe89063570
per_cpu_offset[3]:0x7ffe89063590
0x10
0x11
0x12
0x13
0xf10
0xf11
0xf12
0xf13
*/
1、静态Per-CPU变量实现
++1.1、静态Per-CPU的声明/定义++
/*
_per_cpu_offset是一个unsigned long类型的数组,数组索引为CPU编号(cpu),
数组元素值为该CPU的Per-CPU区域相对于全局Per-CPU区域起始地址(__per_cpu_start)的字节偏移量
*/
unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
#define per_cpu_offset(x) (__per_cpu_offset[x])通过链接脚本可知,静态Per-CPU数据存放在一个段中
在vmlinux.lds链接脚本中.data..percpu段信息__per_cpu_start,__per_cpu_end
. = ALIGN((1 << 12));
.data..percpu : AT(ADDR(.data..percpu) - 0) {
__per_cpu_load = .;
__per_cpu_start = .;
*(.data..percpu..first)
. = ALIGN((1 << 12)); /*4K对齐*/
*(.data..percpu..page_aligned)
. = ALIGN((1 << 6)); /*64字节对齐*/
*(.data..percpu..read_mostly)
. = ALIGN((1 << 6)); /*64字节对齐*/
*(.data..percpu)
*(.data..percpu..shared_aligned)
__per_cpu_end = .;
}
注意,__per_cpu_load/__per_cpu_start为静态Per_CPU段的起始地址,__per_cpu_end为结束地址。静态Per-CPU变量声明和定义宏定义如下:
/*
* Variant(变体) on the per-CPU variable declaration/definition theme used for ordinary(普通) per-CPU variables.
* 对用于普通每 CPU 变量的每 CPU 变量声明/定义模式的一个变体
*/
/*声明变量在".data..percpu" 段中*/
#define DECLARE_PER_CPU(type, name) \
DECLARE_PER_CPU_SECTION(type, name, "")
/*定义变量在".data..percpu" 段中*/
#define DEFINE_PER_CPU(type, name) \
DEFINE_PER_CPU_SECTION(type, name, "")
/*
* Normal declaration and definition macros.
*/
/*声明变量在".data..percpu" 段中*/
#define DECLARE_PER_CPU_SECTION(type, name, sec) \
extern/*声明*/ __PCPU_ATTRS(sec) __typeof__(type)/*类型*/ name/*变量名*/
/*定义变量在".data..percpu" 段中*/
#define DEFINE_PER_CPU_SECTION(type, name, sec) \
__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES \
__typeof__(type)/*类型*/ name/*变量名*/
#endif
# define __percpu
#define PER_CPU_BASE_SECTION ".data..percpu"
#define PER_CPU_ATTRIBUTES
#define PER_CPU_DEF_ATTRIBUTES
/*
#define PER_CPU_BASE_SECTION ".data..percpu"
__PCPU_ATTRS 设置定义的变量在".data..percpu"段中
*/
#define __PCPU_ATTRS(sec) \
__percpu __attribute__((section(PER_CPU_BASE_SECTION/*".data..percpu"*/ sec))) \
PER_CPU_ATTRIBUTES++1.2、静态Per-CPU的使用++
this_cpu_ptr(&var),获取当前CPU的变量指针,内部(内核配套操作this_cpu_read(),this_cpu_add())会隐式禁用抢占
this_cpu_read(var),读取当前CPU的变量值,原子操作,禁用抢占
this_cpu_add(var,val):给当前CPU变量加值,原子操作,禁用抢占
per_cpu(var,cpu),访问指定CPU的变量,无安全保护,需手动加锁
++1.2.1、this_cpu_ptr(&var)函数实现分析++
this_cpu_ptr宏的核心作用是计算当前CPU的Per-CPU变量副本地址,其本身并不直接包含禁用抢占的代码。但使用this_cpu_ptr进行读-修改-写(RMW)操作时,内核配套的宏(如this_cpu_inc、this_cpu_add等)会通过preempt_disable()/preempt_enable()隐式禁用抢占,以确保操作的原子性(相对于当前CPU的抢占）
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
#define raw_cpu_ptr(ptr) \
({ \
__verify_pcpu_ptr(ptr); \
arch_raw_cpu_ptr(ptr); \
})
/*
__verify_pcpu_ptr检查传入的ptr是否是带__percpu属性的指针(即Per-CPU 指针),
如果不是，编译器会抛出类型不兼容的警告/错误,提前拦截错误的Per-CPU指针使用
*/
#define __verify_pcpu_ptr(ptr) \
do { \
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
(void)__vpp_verify; \
} while (0)继续分析实现过程：
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
#define per_cpu_offset(x) (__per_cpu_offset[x])
#define SHIFT_PERCPU_PTR(__p, __offset) \
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
# define RELOC_HIDE(ptr, off) \
({ unsigned long __ptr; \
__ptr = (unsigned long) (ptr); \
(typeof(ptr)) (__ptr + (off)); })++1.2.2、this_cpu_read/this_cpu_write实现分析++
include/linux/percpu-defs.h
#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_/*注意最后有个下划线(_)*/, pcp)
#define __pcpu_size_call_return(stem, variable) \
({ \
typeof(variable) pscr_ret__; \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: pscr_ret__ = stem##1(variable); break; \
case 2: pscr_ret__ = stem##2(variable); break; \
case 4: pscr_ret__ = stem##4(variable); break; \
case 8: pscr_ret__ = stem##8(variable); break; \
default: \
__bad_size_call_parameter(); break; \
} \
pscr_ret__; \
})
/include/asm-generic/percpu.h
#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
#define this_cpu_generic_read(pcp) \
({ \
typeof(pcp) __ret; \
preempt_disable(); \
__ret = *this_cpu_ptr(&(pcp)); \
preempt_enable(); \
__ret; \
})
__ret = *this_cpu_ptr(&(pcp));在关闭内核抢占的情况下直接对内存地址进行解应用获取值。
preempt_disable()/preempt_enable():关闭和开始抢占。
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
#define __pcpu_size_call(stem, variable, ...) \
do { \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: stem##1(variable, __VA_ARGS__);break; \
case 2: stem##2(variable, __VA_ARGS__);break; \
case 4: stem##4(variable, __VA_ARGS__);break; \
case 8: stem##8(variable, __VA_ARGS__);break; \
default: \
__bad_size_call_parameter();break; \
} \
} while (0)
#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#define this_cpu_generic_to_op(pcp, val, op) \
do { \
unsigned long __flags; \
raw_local_irq_save(__flags); \
*raw_cpu_ptr(&(pcp)) op val; /**raw_cpu_ptr(&(pcp)) = val;*/ \
raw_local_irq_restore(__flags); \
} while (0)
#define raw_cpu_ptr(ptr) \
({ \
__verify_pcpu_ptr(ptr); \
arch_raw_cpu_ptr(ptr); \
})++2、动态Per-CPU变量实现++
++2.1、动态Per-CPU访问接口++
alloc_percpu:高层动态分配(推荐)
free_percpu:高层动态释放(推荐),释放 alloc_percpu分配的动态 Per-CPU 变量，自动回收 chunk 空间（若所有副本均释放）
this_cpu_ptr(ptr):获取当前CPU的ptr副本地址(ptr为动态Per-CPU指针)
per_cpu_ptr(ptr,cpu):获取指定cpu的ptr副本地址(跨CPU访问,需谨慎)
get_cpu_ptr(ptr):获取当前CPU副本地址,禁用抢占(用于多步操作,需配对put_cpu_ptr）
put_cpu_ptr(ptr):恢复抢占（与 get_cpu_ptr配对）
++2.1.1、get_cpu_ptr/put_cpu_ptr实现++
#define per_cpu_ptr(ptr, cpu) \
({ \
__verify_pcpu_ptr(ptr); \
SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
})
/*get_cpu_ptr与put_cpu_ptr配对使用*/
#define get_cpu_ptr(var) \
({ \
preempt_disable(); \
this_cpu_ptr(var); \
})
#define put_cpu_ptr(var) \
do { \
(void)(var); /*消除警告的作用*/ \
preempt_enable(); \
} while (0)++2.1.2、alloc_percpu实现++
分配动态Per-CPU变量
/include/linux/percpu.h
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), \
__alignof__(type))
/mm/percpu.c
/**
* __alloc_percpu - allocate dynamic percpu area
* @size: size of area to allocate in bytes
* @align: alignment of area (max PAGE_SIZE)
*
* Equivalent to __alloc_percpu_gfp(size, align, %GFP_KERNEL).
*/
void __percpu *__alloc_percpu(size_t size, size_t align)
{
return pcpu_alloc(size, align, false, GFP_KERNEL);
}++2.1.3、free_percpu实现++
释放动态Per-CPU变量
void free_percpu(void __percpu *ptr)