目录
[eBPF in Android](#eBPF in Android)
[Android eBPF kprobe dma代码](#Android eBPF kprobe dma代码)
[定义一个 PROG](#定义一个 PROG)
bpfprogs/memstats/MemStats.cpp
bpfprogs/memstats/MemStatsMain.cpp
eBPF in Android
官网:https://source.android.com/docs/core/architecture/kernel/bpf?hl=zh-cn#tracepoints
Andoid官方网站有比较详细的介绍,在Android中使用eBPF的官方示例,可以参考以上链接。
Android eBPF kprobe dma代码
在 源目录/system/bpfprogs,可以开发我们自己的 eBPF程序。
在system/bpfprogs 源码目录下,新建一个 memStats.c 示例程序,在次程序中有几个关键点,下面我逐一进行讲解。
memStats.c
cpp
#include <bpf_helpers.h>
#include <sys/types.h>
struct pt_regs {
unsigned long long regs[31];
unsigned long long sp;
unsigned long long pc;
unsigned long long pstate;
};
#define DMABUF_MAP_SIZE 4096
struct dmabuf_info {
unsigned long pid;
uint64_t inode;
char comm[16];
uint64_t size;
};
DEFINE_BPF_MAP_GRW(dmabuf_mem_map, HASH, uint64_t, struct dmabuf_info, DMABUF_MAP_SIZE,
AID_SYSTEM);
DEFINE_BPF_PROG("kprobe/dmabuf_setup", AID_ROOT, AID_SYSTEM, kp_dmabuf_setup)
(struct pt_regs* regs) {
const int ALLOW = 1;
struct dmabuf_info cur_val = { 0 };
unsigned long long tempAddr;
size_t size;
unsigned long inode;
pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff;
bpf_probe_read(&size, sizeof(size), (void*)regs->regs[0]); // size = [x0]; dmabuf->size
bpf_probe_read(&tempAddr, sizeof(tempAddr), ((void*)regs->regs[1] + 32)); // tempAddr = [x1 + 32];
bpf_probe_read(&inode, sizeof(inode), ((void*)tempAddr + 64)); // inode = [[x1 + 32] + 64];
cur_val.pid = pid;
cur_val.size = size;
cur_val.inode = inode;
bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm));
bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY);
return ALLOW;
}
LICENSE("GPL");
定义一个MAP
cpp
DEFINE_BPF_MAP_GRW(dmabuf_mem_map, HASH, uint64_t, struct dmabuf_info, DMABUF_MAP_SIZE,
AID_SYSTEM);
其中:
-
MAP的名字为:dmabuf_mem_map
-
MAP使用HASH进行存储,常见的还有 ARRAY。
-
MAP的key的类型为 uint64_t,value的类型为 struct dmabuf_info。后续会使用 这个函数 bpf_dmabuf_mem_map_update_elem(&key, &value, flags) 进行key和value的映射。
-
MAP的大小 DMABUF_MAP_SIZE 设置为 4096
-
权限是 AID_SYSTEM
定义一个 PROG
cpp
DEFINE_BPF_PROG("kprobe/dmabuf_setup", AID_ROOT, AID_SYSTEM, kp_dmabuf_setup)
(struct pt_regs* regs) {
const int ALLOW = 1;
struct dmabuf_info cur_val = { 0 };
unsigned long long tempAddr;
size_t size;
unsigned long inode;
pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff;
bpf_probe_read(&size, sizeof(size), (void*)regs->regs[0]); // size = [x0]; dmabuf->size
bpf_probe_read(&tempAddr, sizeof(tempAddr), ((void*)regs->regs[1] + 32)); // tempAddr = [x1 + 32];
bpf_probe_read(&inode, sizeof(inode), ((void*)tempAddr + 64)); // inode = [[x1 + 32] + 64];
cur_val.pid = pid;
cur_val.size = size;
cur_val.inode = inode;
bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm));
bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY);
return ALLOW;
}
其中:
-
pid_t pid = (bpf_get_current_pid_tgid() >> 32) & 0xffffffff; 获取进程的 pid号
-
三个 bpf_probe_read(...) 获取对应的 dmabuf的 size 和 inode。这个比较那难理解,我这边详细解释一下。首先需要知道 关于aarch64调用传入参数规则:
怎样获取kprobe对应函数的参数? ARM64中参数1~参数8 分别保存到 X0~X7 寄存器中 即x0存储参数1.....
详细参考这篇文章:第16部分- Linux ARM汇编 ARM64调用标准 - 掘金 -
那么寄存器中的参数偏移是怎么样计算得到的呢?主要是通过 gdb 调试得到的。
cpp
怎么获取函数输入变量偏移?
aarch64-linux-android-gdb vmlinux
(gdb) ptype/T struct dma_buf
type = struct dma_buf {
size_t size;
struct file *file;
struct list_head attachments;
const struct dma_buf_ops *ops;
struct mutex lock;
unsigned int vmapping_counter;
struct iosys_map vmap_ptr;
const char *exp_name;
const char *name;
spinlock_t name_lock;
struct module *owner;
struct list_head list_node;
void *priv;
struct dma_resv *resv;
wait_queue_head_t poll;
struct dma_buf_poll_cb_t cb_in;
struct dma_buf_poll_cb_t cb_out;
struct dma_buf_sysfs_entry *sysfs_entry;
u64 android_kabi_reserved1;
u64 android_kabi_reserved2;
}
(gdb) print (int)&((struct dma_buf *)0)->name
$2 = 120
(gdb) print (int)&((struct dma_buf *)0)->exp_name
$3 = 112
(gdb) print (int)&((struct dma_buf *)0)->file
$6 = 8
bpf_get_current_comm(cur_val.comm, sizeof(cur_val.comm)); 获得进程的名字。
bpf_dmabuf_mem_map_update_elem(&(cur_val.inode), &cur_val, BPF_ANY); 建立 key 和 value的 MAP映射。
bpfprogs/Android.bp
需要在 Android.bp中添加以下代码:
cpp
bpf {
name: "memStats.o",
srcs: ["memStats.c"],
btf: true,
cflags: [
"-Wall",
"-Werror",
],
}
怎么样写一个测试程序,验证上面的 eBPF程序是没有没有问题的呢?
测试程序
主要参考安卓开源代码: frameworks/native/services/gpuservice/ 中的实现部分。
bpfprogs/memstats/Android.bp
cpp
// Copyright 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package {
default_applicable_licenses: ["system_bpfprogs_license"],
}
cc_binary {
name: "memStats",
srcs: [
"MemStatsMain.cpp",
"MemStats.cpp",
],
header_libs: ["bpf_headers"],
shared_libs: [
"libbase",
"libbpf_bcc",
"libcutils",
"liblog",
"libutils",
],
export_header_lib_headers: ["bpf_headers"],
export_shared_lib_headers: ["libbase"],
cppflags: [
"-Wall",
"-Werror",
"-Wformat",
"-Wthread-safety",
"-Wunused",
"-Wunreachable-code",
],
}
bpfprogs/memstats/MemStats.h
cpp
/*
* Copyright 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <bpf/BpfMap.h>
#include <utils/String16.h>
#include <utils/Vector.h>
#include <functional>
class MemStats {
public:
MemStats() = default;
~MemStats();
// initialize eBPF program and map
void initialize();
bool isInitialized() { return mInitialized.load(); }
private:
std::atomic<bool> mInitialized = false;
// tracepoint event category
// tracepoint event category
static constexpr char * kMemStatsKprobeTraceGroup[] = {"kprobes"};
// tracepoint
static constexpr char * kMemStatsDmabufKprobe[] ={"dmabuf_setup"};
// pinned bpf c program path in bpf sysfs
static constexpr char * kMemStatsDmabufProgPath[] =
{"/sys/fs/bpf/prog_memStats_kprobe_dmabuf_setup"};
// 30 seconds timeout for trying to attach bpf program to tracepoint
static constexpr int kWaitTimeout = 30;
};
bpfprogs/memstats/MemStats.cpp
cpp
/*
* Copyright 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#undef LOG_TAG
#define LOG_TAG "MemStats"
#define ATRACE_TAG ATRACE_TAG_GRAPHICSS3
#include "MemStats.h"
#include <android-base/stringprintf.h>
#include <libbpf.h>
#include <bpf/WaitForProgsLoaded.h>
#include <log/log.h>
#include <unistd.h>
#include <utils/Timers.h>
#include <utils/Trace.h>
#include <unordered_map>
#include <vector>
#include <android-base/file.h>
MemStats::~MemStats() {
for (int i = 0; i < sizeof(kMemStatsDmabufKprobe)/sizeof(char *); i++)
{
bpf_detach_tracepoint(kMemStatsKprobeTraceGroup[0], kMemStatsDmabufKprobe[i]);
}
}
void MemStats::initialize() {
int count = 0;
int fd = 0;
// Make sure bpf programs are loaded
android::bpf::waitForProgsLoaded();
errno = 0;
android::base::WriteStringToFile("p:dmabuf_setup dma_buf_stats_setup","/sys/kernel/debug/tracing/kprobe_events");
for (int i = 0; i < sizeof(kMemStatsDmabufProgPath)/sizeof(char *); i++)
{
fd = android::bpf::retrieveProgram(kMemStatsDmabufProgPath[i]);
if (fd < 0) {
ALOGE("Failed to retrieve pinned program from %s [%d(%s)]", kMemStatsDmabufProgPath[i], errno,
strerror(errno));
}
// Attach the program to the tracepoint, and the tracepoint is automatically enabled here.
errno = 0;
count = 0;
while (bpf_attach_tracepoint(fd, kMemStatsKprobeTraceGroup[0], kMemStatsDmabufKprobe[i]) < 0) {
if (++count > kWaitTimeout) {
ALOGE("Failed to attach bpf program to %s/%s tracepoint [%d(%s)]", kMemStatsKprobeTraceGroup[0],
kMemStatsDmabufKprobe[i], errno, strerror(errno));
}
// Retry until loaded or timeout.
sleep(1);
}
}
mInitialized.store(true);
}
其中:
cpp
android::base::WriteStringToFile("p:dmabuf_setup dma_buf_stats_setup","/sys/kernel/debug/tracing/kprobe_events");
在 bpf_attach_tracepoint(...)之前,将 "p:dmabuf_setup dma_buf_stats_setup" 写入到 "/sys/kernel/debug/tracing/kprobe_events" 这个事件中去。如果不使用这行代码,也可以在 Linux终端使用,adb shell; echo 'p:dmabuf_setup dma_buf_stats_setup' > /sys/kernel/debug/tracing/kprobe_events
bpfprogs/memstats/MemStatsMain.cpp
cpp
#include "MemStats.h"
#include <unistd.h>
int main()
{
class MemStats *memHandle = new MemStats;
memHandle->initialize();
while(1)
{
sleep(10);
}
return 0;
}
编译运行
-
make memStats.o
-
adb push system/etc/bpf/memStats.o /system/etc/bpf/
-
adb reboot; #重启之后就可以在 /sys/fs/bpf/ 目录下,看到生成了 这个文件 map_memStats_dmabuf_mem_map
-
make memStats
-
adb push memStats /data/local/tmp/
-
adb shell ; /data/local/tmp/memStats
-
在另外一个Linux终端:cat /sys/fs/bpf/map_memStats_dmabuf_mem_map
可以看到输出以下信息:
cpp
/ # cat /sys/fs/bpf/map_memStats_dmabuf_mem_map
# WARNING!! The output is for debug purpose only
# WARNING!! The output format will change
697: {1512,697,['b','i','n','d','e','r',':','1','5','1','2','_','3',],188416,}
1344: {1512,1344,['P','r','e','v','i','e','w','_','4',],22020096,}
752: {1512,752,['P','r','e','v','i','e','w','_','0',],4096,}
589: {1525,589,['b','i','n','d','e','r',':','1','5','2','5','_','1',],10522624,}
1175: {1512,1175,['P','r','e','v','i','e','w','_','2',],462848,}
1498: {1512,1498,['P','r','e','v','i','e','w','_','4',],4096,}
1539: {1525,1539,['b','i','n','d','e','r',':','1','5','2','5','_','1',],73728,}
712: {1512,712,['P','r','e','v','i','e','w','_','2',],4096,}
1469: {1525,1469,['b','i','n','d','e','r',':','1','5','2','5','_','1',],2506752,}
1520: {2553,1520,['.','v','o','r','b','i','s','.','d','e','c','o','d','e','r',],32768,}
745: {1512,745,['P','r','e','v','i','e','w','_','1',],4096,}
1137: {1512,1137,['P','r','e','v','i','e','w','_','7',],4096,}
1186: {1512,1186,['b','i','n','d','e','r',':','1','5','1','2','_','3',],6451200,}
724: {1512,724,['P','r','e','v','i','e','w','_','5',],4096,}
791: {1512,791,['P','r','e','v','i','e','w','_','7',],4096,}
1159: {1512,1159,['P','r','e','v','i','e','w','_','4',],462848,}
810: {1512,810,['P','r','e','v','i','e','w','_','7',],4096,}
1285: {1512,1285,['P','r','e','v','i','e','w','_','0',],4096,}
761: {1512,761,['P','r','e','v','i','e','w','_','8',],4096,}
735: {1512,735,['P','r','e','v','i','e','w','_','1',],4096,}
630: {1512,630,['b','i','n','d','e','r',':','1','5','1','2','_','3',],4096,}
1012: {1512,1012,['b','i','n','d','e','r',':','1','5','1','2','_','3',],45056,}
结果分析
解释以下输出结果,以这个为例子:697: {1512,697,['b','i','n','d','e','r',':','1','5','1','2','_','3',],188416,}
其中 697 是 inode的值,{1512,697,['b','i','n','d','e','r',':','1','5','1','2','_','3',],188416,} 是struct dmabuf_info 结构体的输出。具体输出每一项为对应结构体中的值,如下所示:
cpp
struct dmabuf_info {
unsigned long pid; //1512
uint64_t inode; //697
char comm[16]; //['b','i','n','d','e','r',':','1','5','1','2','_','3',]
uint64_t size; //188416
};
小结
以上是一个具体的 eBPF实例程序。通过 kprobe 监控内核中的 dma_buf_stats_setup 接口。
kernel_platform/common/drivers/dma-buf/dma-buf-sysfs-stats.c
cpp
int dma_buf_stats_setup(struct dma_buf *dmabuf, struct file *file)
172 {
173 struct dma_buf_sysfs_entry *sysfs_entry;
174 int ret;
175
176 if (!dmabuf->exp_name) {
177 pr_err("exporter name must not be empty if stats needed\n");
178 return -EINVAL;
179 }
180
181 sysfs_entry = kzalloc(sizeof(struct dma_buf_sysfs_entry), GFP_KERNEL);
182 if (!sysfs_entry)
183 return -ENOMEM;
184
185 sysfs_entry->kobj.kset = dma_buf_per_buffer_stats_kset;
186 sysfs_entry->dmabuf = dmabuf;
187
188 dmabuf->sysfs_entry = sysfs_entry;
189
190 /* create the directory for buffer stats */
191 ret = kobject_init_and_add(&sysfs_entry->kobj, &dma_buf_ktype, NULL,
192 "%lu", file_inode(file)->i_ino);
193 if (ret)
194 goto err_sysfs_dmabuf;
195
196 return 0;
197
198 err_sysfs_dmabuf:
199 kobject_put(&sysfs_entry->kobj);
200 dmabuf->sysfs_entry = NULL;
201 return ret;
202 }
后续将在 Android上输出更多的 eBPF demo程序。