Android中eBPF使用原理以及 kprobe dmabuf_setup实例

目录

[eBPF in Android](#eBPF in Android)

[Android eBPF kprobe dma代码](#Android eBPF kprobe dma代码)

定义一个MAP

[定义一个 PROG](#定义一个 PROG)

bpfprogs/Android.bp

测试程序

bpfprogs/memstats/Android.bp

bpfprogs/memstats/MemStats.h

bpfprogs/memstats/MemStats.cpp

bpfprogs/memstats/MemStatsMain.cpp

编译运行

结果分析

小结

---

eBPF in Android

官网：https://source.android.com/docs/core/architecture/kernel/bpf?hl=zh-cn#tracepoints

Andoid官方网站有比较详细的介绍，在Android中使用eBPF的官方示例，可以参考以上链接。

Android eBPF kprobe dma代码

在 源目录/system/bpfprogs，可以开发我们自己的 eBPF程序。

在system/bpfprogs 源码目录下，新建一个 memStats.c 示例程序，在次程序中有几个关键点，下面我逐一进行讲解。

memStats.c

#include <bpf_helpers.h>
#include <sys/types.h>


struct pt_regs {
    unsigned long long regs[31];
    unsigned long long sp;
    unsigned long long pc;
    unsigned long long pstate;
};

#define DMABUF_MAP_SIZE 4096

struct dmabuf_info {
    unsigned long pid;
    uint64_t inode;
    char comm[16];
    uint64_t size;
};

DEFINE_BPF_MAP_GRW(dmabuf_mem_map, HASH, uint64_t, struct dmabuf_info, DMABUF_MAP_SIZE,
                   AID_SYSTEM);

DEFINE_BPF_PROG("kprobe/dmabuf_setup", AID_ROOT, AID_SYSTEM, kp_dmabuf_setup)
(struct pt_regs* regs) {
    const int ALLOW = 1;
    struct dmabuf_info cur_val = { 0 };
    unsigned long long tempAddr;
    size_t size;
    unsigned long inode;

    pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff;

    bpf_probe_read(&size, sizeof(size), (void*)regs->regs[0]); // size = [x0]; dmabuf->size
    bpf_probe_read(&tempAddr, sizeof(tempAddr), ((void*)regs->regs[1] + 32)); // tempAddr = [x1 + 32];
    bpf_probe_read(&inode, sizeof(inode), ((void*)tempAddr + 64)); // inode = [[x1 + 32] + 64];


    cur_val.pid = pid;
    cur_val.size = size;
    cur_val.inode = inode;

    bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm));
    bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY);
    return ALLOW;
}

LICENSE("GPL");

定义一个MAP

DEFINE_BPF_MAP_GRW(dmabuf_mem_map, HASH, uint64_t, struct dmabuf_info, DMABUF_MAP_SIZE,
                   AID_SYSTEM);

其中：

1. MAP的名字为：dmabuf_mem_map

2. MAP使用HASH进行存储，常见的还有 ARRAY。

3. MAP的key的类型为 uint64_t，value的类型为 struct dmabuf_info。后续会使用 这个函数 bpf_dmabuf_mem_map_update_elem(&key, &value, flags) 进行key和value的映射。

4. MAP的大小 DMABUF_MAP_SIZE 设置为 4096

5. 权限是 AID_SYSTEM

定义一个 PROG

DEFINE_BPF_PROG("kprobe/dmabuf_setup", AID_ROOT, AID_SYSTEM, kp_dmabuf_setup)
(struct pt_regs* regs) {
    const int ALLOW = 1;
    struct dmabuf_info cur_val = { 0 };
    unsigned long long tempAddr;
    size_t size;
    unsigned long inode;

    pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff;

    bpf_probe_read(&size, sizeof(size), (void*)regs->regs[0]); // size = [x0]; dmabuf->size
    bpf_probe_read(&tempAddr, sizeof(tempAddr), ((void*)regs->regs[1] + 32)); // tempAddr = [x1 + 32];
    bpf_probe_read(&inode, sizeof(inode), ((void*)tempAddr + 64)); // inode = [[x1 + 32] + 64];


    cur_val.pid = pid;
    cur_val.size = size;
    cur_val.inode = inode;

    bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm));
    bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY);
    return ALLOW;
}

其中：

1. pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff; 获取进程的 pid号

2. 三个 bpf_probe_read(...) 获取对应的 dmabuf的 size 和 inode。这个比较那难理解，我这边详细解释一下。首先需要知道 关于aarch64调用传入参数规则：

   怎样获取kprobe对应函数的参数？ ARM64中参数1~参数8 分别保存到 X0~X7 寄存器中 即x0存储参数1..... 详细参考这篇文章：第16部分- Linux ARM汇编 ARM64调用标准 - 掘金

3. 那么寄存器中的参数偏移是怎么样计算得到的呢？主要是通过 gdb 调试得到的。

怎么获取函数输入变量偏移？
 aarch64-linux-android-gdb vmlinux
(gdb) ptype/T struct dma_buf
type = struct dma_buf {
    size_t size;
    struct file *file;
    struct list_head attachments;
    const struct dma_buf_ops *ops;
    struct mutex lock;
    unsigned int vmapping_counter;
    struct iosys_map vmap_ptr;
    const char *exp_name;
    const char *name;
    spinlock_t name_lock;
    struct module *owner;
    struct list_head list_node;
    void *priv;
    struct dma_resv *resv;
    wait_queue_head_t poll;
    struct dma_buf_poll_cb_t cb_in;
    struct dma_buf_poll_cb_t cb_out;
    struct dma_buf_sysfs_entry *sysfs_entry;
    u64 android_kabi_reserved1;
    u64 android_kabi_reserved2;
}
(gdb)  print (int)&((struct dma_buf *)0)->name
$2 = 120
(gdb)  print (int)&((struct dma_buf *)0)->exp_name
$3 = 112
(gdb)  print (int)&((struct dma_buf *)0)->file
$6 = 8

bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm)); 获得进程的名字。

bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY); 建立 key 和 value的 MAP映射。

bpfprogs/Android.bp

需要在 Android.bp中添加以下代码：

bpf {
    name: "memStats.o",
    srcs: ["memStats.c"],
    btf: true,
    cflags: [
        "-Wall",
        "-Werror",
    ],
}

怎么样写一个测试程序，验证上面的 eBPF程序是没有没有问题的呢？

测试程序

主要参考安卓开源代码： frameworks/native/services/gpuservice/ 中的实现部分。

bpfprogs/memstats/Android.bp

// Copyright 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package {
    default_applicable_licenses: ["system_bpfprogs_license"],
}
cc_binary {
    name: "memStats",
    srcs: [
        "MemStatsMain.cpp",
        "MemStats.cpp",
    ],
    header_libs: ["bpf_headers"],
    shared_libs: [
        "libbase",
        "libbpf_bcc",
        "libcutils",
        "liblog",
        "libutils",
    ],
    export_header_lib_headers: ["bpf_headers"],
    export_shared_lib_headers: ["libbase"],
    cppflags: [
        "-Wall",
        "-Werror",
        "-Wformat",
        "-Wthread-safety",
        "-Wunused",
        "-Wunreachable-code",
    ],
}

bpfprogs/memstats/MemStats.h

/*
 * Copyright 2020 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#pragma once
#include <bpf/BpfMap.h>
#include <utils/String16.h>
#include <utils/Vector.h>
#include <functional>

class MemStats {
public:
    MemStats() = default;
    ~MemStats();
    // initialize eBPF program and map
    void initialize();
    bool isInitialized() { return mInitialized.load(); }
private:
    std::atomic<bool> mInitialized = false;
    // tracepoint event category

    // tracepoint event category
    static constexpr char * kMemStatsKprobeTraceGroup[] = {"kprobes"};

    // tracepoint
    static constexpr char * kMemStatsDmabufKprobe[] ={"dmabuf_setup"};

    // pinned bpf c program path in bpf sysfs
    static constexpr char * kMemStatsDmabufProgPath[] =
    {"/sys/fs/bpf/prog_memStats_kprobe_dmabuf_setup"};

    // 30 seconds timeout for trying to attach bpf program to tracepoint
    static constexpr int kWaitTimeout = 30;
};

bpfprogs/memstats/MemStats.cpp

/*
 * Copyright 2020 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#undef LOG_TAG
#define LOG_TAG "MemStats"
#define ATRACE_TAG ATRACE_TAG_GRAPHICSS3
#include "MemStats.h"
#include <android-base/stringprintf.h>
#include <libbpf.h>
#include <bpf/WaitForProgsLoaded.h>
#include <log/log.h>
#include <unistd.h>
#include <utils/Timers.h>
#include <utils/Trace.h>
#include <unordered_map>
#include <vector>

#include <android-base/file.h>


MemStats::~MemStats() {

    for (int i = 0; i < sizeof(kMemStatsDmabufKprobe)/sizeof(char *); i++)
    {
        bpf_detach_tracepoint(kMemStatsKprobeTraceGroup[0], kMemStatsDmabufKprobe[i]);
    }
}

void MemStats::initialize() {
    int count = 0;
    int fd = 0;
    // Make sure bpf programs are loaded
    android::bpf::waitForProgsLoaded();
    errno = 0;

    android::base::WriteStringToFile("p:dmabuf_setup dma_buf_stats_setup","/sys/kernel/debug/tracing/kprobe_events");
    
    for (int i = 0; i < sizeof(kMemStatsDmabufProgPath)/sizeof(char *); i++)
    {
        fd = android::bpf::retrieveProgram(kMemStatsDmabufProgPath[i]);

        if (fd < 0) {
            ALOGE("Failed to retrieve pinned program from %s [%d(%s)]", kMemStatsDmabufProgPath[i], errno,
                strerror(errno));
        }
        // Attach the program to the tracepoint, and the tracepoint is automatically enabled here.
        errno = 0;
        count = 0;
        while (bpf_attach_tracepoint(fd, kMemStatsKprobeTraceGroup[0], kMemStatsDmabufKprobe[i]) < 0) {
            
            if (++count > kWaitTimeout) {
                ALOGE("Failed to attach bpf program to %s/%s tracepoint [%d(%s)]", kMemStatsKprobeTraceGroup[0],
                    kMemStatsDmabufKprobe[i], errno, strerror(errno));
            }
            // Retry until loaded or timeout.
            sleep(1);
        }
    }

    mInitialized.store(true);
}

其中：

android::base::WriteStringToFile("p:dmabuf_setup dma_buf_stats_setup","/sys/kernel/debug/tracing/kprobe_events");

在 bpf_attach_tracepoint(...)之前，将 "p:dmabuf_setup dma_buf_stats_setup" 写入到 "/sys/kernel/debug/tracing/kprobe_events" 这个事件中去。如果不使用这行代码，也可以在 Linux终端使用，adb shell; echo 'p:dmabuf_setup dma_buf_stats_setup' > /sys/kernel/debug/tracing/kprobe_events

bpfprogs/memstats/MemStatsMain.cpp

#include "MemStats.h"
#include <unistd.h>
int main()
{
    class MemStats *memHandle = new MemStats;

    memHandle->initialize();
    
    while(1)
    {
        sleep(10);
    }

 return 0;
}

编译运行

1. make memStats.o

2. adb push system/etc/bpf/memStats.o /system/etc/bpf/

3. adb reboot; #重启之后就可以在 /sys/fs/bpf/ 目录下，看到生成了 这个文件 map_memStats_dmabuf_mem_map

4. make memStats

5. adb push memStats /data/local/tmp/

6. adb shell ; /data/local/tmp/memStats

7. 在另外一个Linux终端：cat /sys/fs/bpf/map_memStats_dmabuf_mem_map

可以看到输出以下信息：

/ # cat  /sys/fs/bpf/map_memStats_dmabuf_mem_map
# WARNING!! The output is for debug purpose only
# WARNING!! The output format will change
697: {1512,697,['b','i','n','d','e','r',':','1','5','1','2','_','3',],188416,}
1344: {1512,1344,['P','r','e','v','i','e','w','_','4',],22020096,}
752: {1512,752,['P','r','e','v','i','e','w','_','0',],4096,}
589: {1525,589,['b','i','n','d','e','r',':','1','5','2','5','_','1',],10522624,}
1175: {1512,1175,['P','r','e','v','i','e','w','_','2',],462848,}
1498: {1512,1498,['P','r','e','v','i','e','w','_','4',],4096,}
1539: {1525,1539,['b','i','n','d','e','r',':','1','5','2','5','_','1',],73728,}
712: {1512,712,['P','r','e','v','i','e','w','_','2',],4096,}
1469: {1525,1469,['b','i','n','d','e','r',':','1','5','2','5','_','1',],2506752,}
1520: {2553,1520,['.','v','o','r','b','i','s','.','d','e','c','o','d','e','r',],32768,}
745: {1512,745,['P','r','e','v','i','e','w','_','1',],4096,}
1137: {1512,1137,['P','r','e','v','i','e','w','_','7',],4096,}
1186: {1512,1186,['b','i','n','d','e','r',':','1','5','1','2','_','3',],6451200,}
724: {1512,724,['P','r','e','v','i','e','w','_','5',],4096,}
791: {1512,791,['P','r','e','v','i','e','w','_','7',],4096,}
1159: {1512,1159,['P','r','e','v','i','e','w','_','4',],462848,}
810: {1512,810,['P','r','e','v','i','e','w','_','7',],4096,}
1285: {1512,1285,['P','r','e','v','i','e','w','_','0',],4096,}
761: {1512,761,['P','r','e','v','i','e','w','_','8',],4096,}
735: {1512,735,['P','r','e','v','i','e','w','_','1',],4096,}
630: {1512,630,['b','i','n','d','e','r',':','1','5','1','2','_','3',],4096,}
1012: {1512,1012,['b','i','n','d','e','r',':','1','5','1','2','_','3',],45056,}

结果分析

解释以下输出结果，以这个为例子：697: {1512,697,'b','i','n','d','e','r',':','1','5','1','2','_','3',,188416,}

其中 697 是 inode的值，{1512,697,'b','i','n','d','e','r',':','1','5','1','2','_','3',,188416,} 是struct dmabuf_info 结构体的输出。具体输出每一项为对应结构体中的值，如下所示：

struct dmabuf_info {
    unsigned long pid;  //1512
    uint64_t inode;     //697
    char comm[16];      //['b','i','n','d','e','r',':','1','5','1','2','_','3',]
    uint64_t size;      //188416
};

小结

以上是一个具体的 eBPF实例程序。通过 kprobe 监控内核中的 dma_buf_stats_setup 接口。

kernel_platform/common/drivers/dma-buf/dma-buf-sysfs-stats.c

int dma_buf_stats_setup(struct dma_buf *dmabuf, struct file *file)
172  {
173  	struct dma_buf_sysfs_entry *sysfs_entry;
174  	int ret;
175  
176  	if (!dmabuf->exp_name) {
177  		pr_err("exporter name must not be empty if stats needed\n");
178  		return -EINVAL;
179  	}
180  
181  	sysfs_entry = kzalloc(sizeof(struct dma_buf_sysfs_entry), GFP_KERNEL);
182  	if (!sysfs_entry)
183  		return -ENOMEM;
184  
185  	sysfs_entry->kobj.kset = dma_buf_per_buffer_stats_kset;
186  	sysfs_entry->dmabuf = dmabuf;
187  
188  	dmabuf->sysfs_entry = sysfs_entry;
189  
190  	/* create the directory for buffer stats */
191  	ret = kobject_init_and_add(&sysfs_entry->kobj, &dma_buf_ktype, NULL,
192  				   "%lu", file_inode(file)->i_ino);
193  	if (ret)
194  		goto err_sysfs_dmabuf;
195  
196  	return 0;
197  
198  err_sysfs_dmabuf:
199  	kobject_put(&sysfs_entry->kobj);
200  	dmabuf->sysfs_entry = NULL;
201  	return ret;
202  }

后续将在 Android上输出更多的 eBPF demo程序。