手动编译了解过程

通过对关键步骤make M=samples/bpf的实践，我们已经可以编译出内核源码中提供的ebpf样例。但这还不够我们充分地理解这个编译过程，我们将这编译过程拆解一下，拆解成可以一步步执行的那种，首先是环境准备：

bash 复制代码

$ sudo apt-get install linux-source
$ cd /usr/src/
$ ls
linux-source-5.15.0.tar.bz2 linux-headers-5.15.0-43-generic linux-source-5.15.0
$ sudo tar -xvf linux-source-5.15.0.tar.bz2 -C /tmp/
$ cd /tmp/linux-source-5.15.0/

内核源码树环境准备

bash 复制代码

$ sudo make oldconfig
$ sudo make init 
$ sudo make headers_install

进入tools目录编译依赖库

bash 复制代码

$ sudo cd tools/lib/bpf/

编译依赖库为object中间件

bash 复制代码

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o libbpf.o libbpf.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o bpf.o bpf.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o btf.o btf.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o nlattr.o nlattr.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o strset.o strset.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o str_error.o str_error.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o hashmap.o hashmap.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o libbpf_probes.o libbpf_probes.c

$ sudo gcc -I. -I/tmp/linux-source-5.15.0/samples/bpf/../..//tools/include -I/tmp/linux-source-5.15.0/samples/bpf/../..//tools/include/uapi -fvisibility=hidden -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -D"BUILD_STR(s)=#s" -c -o /tmp/linux-source-5.15.0/samples/bpf/libbpf/staticobjs/gen_loader.o gen_loader.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o relo_core.o relo_core.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o netlink.o netlink.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o linker.o linker.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o xsk.o xsk.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o ringbuf.o ringbuf.c

$ sudo gcc -g -DHAVE_LIBELF_MMAP_SUPPORT -DCOMPAT_NEED_REALLOCARRAY -fPIC -I. -I/tmp/linux-source-5.15.0/tools/bpf/ -I/tmp/linux-source-5.15.0/tools/arch/x86/include/uapi -I/tmp/linux-source-5.15.0/tools/include/ -I/tmp/linux-source-5.15.0/tools/perf -D"BUILD_STR(s)=#s" -c -o btf_dump.o btf_dump.c

将编译好的object文件链接为libbpf.a静态库

bash 复制代码

$ cd /tmp/linux-source-5.15.0/tools/lib/bpf
$ sudo ar rcs libbpf.a bpf.o btf.o libbpf.o nlattr.o str_error.o strset.o gen_loader.o libbpf_probes.o relo_core.o ringbuf.o linker.o netlink.o xsk.o btf_dump.o

编译用户态测试程序

bash 复制代码

$ cd /tmp/linux-source-5.15.0
$ sudo gcc -I./usr/include -I./tools/testing/selftests/bpf/ -I/tmp/linux-source-5.15.0/samples/bpf/libbpf/include -I./tools/include -I./tools/perf -DHAVE_ATTR_TEST=0  -c -o samples/bpf/trace_output_user.o samples/bpf/trace_output_user.c
$ sudo gcc -I./usr/include -I./tools/testing/selftests/bpf/ -I/tmp/linux-source-5.15.0/samples/bpf/libbpf/include -I./tools/include -I./tools/perf -DHAVE_ATTR_TEST=0 -o samples/bpf/trace_output samples/bpf/trace_output_user.o /tmp/linux-source-5.15.0/samples/bpf/libbpf/libbpf.a -lelf -lz -lrt

编译示例程序的第二部分，即需要给内核虚拟机运行的，由bpf_load()加载的字节码

bash 复制代码

$ cd /tmp/linux-source-5.15.0
$ sudo clang -g -nostdinc -isystem /usr/lib/gcc/x86_64-redhat-linux/8/include -I./arch/x86/include -I./arch/x86/include/generated -I./include -I./arch/x86/include/uapi -I./arch/x86/include/generated/uapi -I./include/uapi -I./include/generated/uapi -include ./include/linux/kconfig.h -Isamples/bpf -Isamples/bpf/libbpf/include/ -Isamples/bpf/libbpf/ -I./tools/testing/selftests/bpf/ -D__KERNEL__ -D__BPF_TRACING__ -D__TARGET_ARCH_x86 -O2 -emit-llvm -c samples/bpf/trace_output_kern.c -o - | sudo llc -march=bpf -filetype=obj -o samples/bpf/trace_output_kern.o
$ sudo llvm-strip -g /tmp/linux-source-5.15.0/samples/bpf/trace_output_kern.o

测试

bash 复制代码

$ cd /tmp/linux-source-5.15.0/samples/bpf
$ sudo ./trace_output
recv 236472 events per sec
100013+0 records in
100013+0 records out
51206656 bytes (51 MB, 49 MiB) copied, 0.420891 s, 122 MB/s

解析源代码

BPF程序结构图：

由上图可知BPF程序要由两部分组成用户态执行程序和BPF bytecode。

trace_output示例的量部分如下，其中trace_output_user.c为用户态执行程序，trace_output_kern.c为BPF bytecode，所以trace_output_user.c使用gcc编译，trace_output_kern.c使用clang编译。

bash 复制代码

$ ls samples/bpf/ | grep trace_output
trace_output_kern.c
trace_output_user.c

trace_output_user.c里的主要内容

cpp 复制代码

int main(int argc, char **argv)
{
        struct perf_buffer_opts pb_opts = {};
        struct bpf_link *link = NULL;
        struct bpf_program *prog;
        struct perf_buffer *pb;
        struct bpf_object *obj;
        int map_fd, ret = 0;
        char filename[256];
        FILE *f;

        snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
        obj = bpf_object__open_file(filename, NULL);
        if (libbpf_get_error(obj)) {
                fprintf(stderr, "ERROR: opening BPF object file failed\n");
                return 0;
        }

        /* load BPF program */
        if (bpf_object__load(obj)) {
                fprintf(stderr, "ERROR: loading BPF object file failed\n");
                goto cleanup;
        }

        map_fd = bpf_object__find_map_fd_by_name(obj, "my_map");
        if (map_fd < 0) {
                fprintf(stderr, "ERROR: finding a map in obj file failed\n");
                goto cleanup;
        }

        prog = bpf_object__find_program_by_name(obj, "bpf_prog1");
        if (libbpf_get_error(prog)) {
                fprintf(stderr, "ERROR: finding a prog in obj file failed\n");
                goto cleanup;
        }

        link = bpf_program__attach(prog);
        if (libbpf_get_error(link)) {
                fprintf(stderr, "ERROR: bpf_program__attach failed\n");
                link = NULL;
                goto cleanup;
        }

        pb_opts.sample_cb = print_bpf_output;
        pb = perf_buffer__new(map_fd, 8, &pb_opts);
        ret = libbpf_get_error(pb);
        if (ret) {
                printf("failed to setup perf_buffer: %d\n", ret);
                return 1;
        }

        f = popen("taskset 1 dd if=/dev/zero of=/dev/null", "r");
        (void) f;

        start_time = time_get_ns();
        while ((ret = perf_buffer__poll(pb, 1000)) >= 0 && cnt < MAX_CNT) {
        }
        kill(0, SIGINT);

cleanup:
        bpf_link__destroy(link);
        bpf_object__close(obj);
        return ret;
}

其中bpf_object__open_file打开的字节码文件trace_output_kern.c里的内容如下：

cpp 复制代码

struct {
        __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
        __uint(key_size, sizeof(int));
        __uint(value_size, sizeof(u32));
        __uint(max_entries, 2);
} my_map SEC(".maps");

SEC("kprobe/" SYSCALL(sys_write))
int bpf_prog1(struct pt_regs *ctx)
{
        struct S {
                u64 pid;
                u64 cookie;
        } data;

        data.pid = bpf_get_current_pid_tgid();
        data.cookie = 0x12345678;

        bpf_perf_event_output(ctx, &my_map, 0, &data, sizeof(data));

        return 0;
}

char _license[] SEC("license") = "GPL";
u32 _version SEC("version") = LINUX_VERSION_CODE;

参考链接

Linux C语言实践eBPF

手动编译了解过程

解析源代码