1. 环境及目标
环境：Ubuntu 24.04.1 LTS
目标：在x86的设备上，调试ARM64的Linux内核
2. qemu的编译安装
2.1 qemu下载
通过网址https://download.qemu.org/进行相应版本选择下载，我们选择比较新的版本，8.2.10进行下载
wget https://download.qemu.org/qemu-8.2.10.tar.xz
tar -xvf qemu-8.2.10.tar.xz
cd qemu-8.2.10/2.2. 安装依赖
2.2.1 配置ubuntu 24.04的源
vim /etc/apt/sources.list.d/ubuntu.sources
Types: deb deb-src
URIs: http://mirrors.ustc.edu.cn/ubuntu/
Suites: noble noble-updates noble-security
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg2.2.2 安装依赖包
apt update
apt install make bison flex build-essential libncurses-dev libssl-dev pkg-config ninja-build libglib2.0-dev libpixman-1-dev libcap-dev libncurses5-dev gcc-arm-linux-gnueabi zlib1g-dev libglib2.0-dev python3-pip slirp libslirp02.2.3 安装sphinx
apt install python3-sphinx python3-msmb-theme python3-dask-sphinx-theme2.2.4 ninja
apt-get install re2c
git clone https://github.com/ninja-build/ninja.git
cd ninja && ./configure.py --bootstrap
cp ./ninja /usr/bin/
ninja --version2.2.5 libcap-ng
virtio-9p (virtfs) on Linux requires libcap-ng-devel and libattr-devel
wget https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/libcap-ng/0.8.5-1/libcap-ng_0.8.5.orig.tar.gz
apt install dh-autoreconf
tar -xvf tar -xvf libcap-ng_0.8.5.orig.tar.gz
cd libcap-ng-0.8.5/
./autogen.sh
./configure
make
make install2.3 qemu编译安装
./configure --target-list=aarch64-softmmu,aarch64-linux-user --enable-slirp
make -j16
make install注：打开--enable-slirp开关对上层组件的主要影响是qemu增加了一种user mode的网络后端实现，该网络后端的实现是在用户态实现的一套tcp/ip协议栈。qemu下-netdev多了一个user的选项参数。user mode的网络简单、独立性好、无需 root 权限、虚拟机网络隔离，但是缺点也很明显：网络性能差、不支持 ICMP 协议，也就 ping 不通、外部网络不能直接访问虚拟机，可以用于一般的测试场景。
3. 内核编译
3.1 下载kernel
git clone https://android.googlesource.com/kernel/common
git checkout -b android15-6.6-pkvm_experimental origin/android15-6.6-pkvm_experimental3.2 交叉编译
交叉编译ARM64 Linux内核，编译完毕后，对应目录会有生成的内核镜像：
ARCH=arm64 make CROSS_COMPILE=aarch64-linux-gnu- O=build menuconfig
Kernel Features
-> [ ] Randomize the address of the kernel image
Device Drivers
-> Block devices
<*> RAM block device support
(16) Default number of RAM disks (NEW)
(65536) Default RAM disk size (kbytes)交叉编译
ARCH=arm64 make CROSS_COMPILE=aarch64-linux-gnu- O=build -j16-
O=build 表示编译的文件输出到build目录，不跟源码混在一起
root@kernel:~/common# file build/arch/arm64/boot/Image
build/arch/arm64/boot/Image: Linux kernel ARM64 boot executable Image, little-endian, 4K pages
4. busybox文件系统制作
参考《aarch64环境下编译kvmtool，基于kvmtool启动最小linux（busybox）》
交叉编译aarch64 BusyBox,安装交叉编译环境依赖
apt-get install gcc-aarch64-linux-gnu下载buysbox源码
wget https://busybox.net/downloads/busybox-1.35.0.tar.bz2
tar -xvf busybox-1.35.0.tar.bz2
cd busybox-1.35.0/把busybox配置为静态编译，这样busybox在运行的时候就不需要额外的动态链接库
-> Settings
-> Build Options
-> Build BusyBox as a static binary (no shared libs)
Networking Utilities -->
[ ] tc 交叉编译BusyBox可执行文件，并输出到_install目录：
ARCH=arm64 make CROSS_COMPILE=aarch64-linux-gnu- menuconfig
ARCH=arm64 make CROSS_COMPILE=aarch64-linux-gnu- -j16
ARCH=arm64 make CROSS_COMPILE=aarch64-linux-gnu- install5. GDB调试
5.1 安装gdb-multiarch
apt install gdb-multiarch5.2 启动虚拟机
qemu-system-aarch64 -nographic -M virt -cpu cortex-a57 -smp 2 -m 4G --kernel common/build/arch/arm64/boot/Image -append "nokaslr root=/dev/ram0 rdinit=/linuxrc console=ttyAMA0" -initrd busybox-1.35.0/rootfs.cpio.gz -s -S - nokaslr 表示关闭地址随机化
- -S 在启动时冻结CPU（使用'c'开始执行）
- -s -gdb tcp::1234的简写
上面的命令会停止，新开一个终端，使用如下命令调试：
cd ~/common/build
# vmlinux 是编译内核时生成的调试文件
gdb-multiarch vmlinux
# 连接 qemu 进行调试：
target remote :1234
# 设置断点
b start_kernel
# 执行内核
c
6. 问题记录
-
libslirp GnuTLS recv error (-110): The TLS connection was non-properly terminated.
Cloning into 'slirp'...
fatal: unable to access 'https://gitlab.freedesktop.org/slirp/libslirp.git/': GnuTLS recv error (-110): The TLS connection was non-properly terminated.../meson.build:945:10: ERROR: Git command failed: ['/usr/bin/git', 'clone', 'https://gitlab.freedesktop.org/slirp/libslirp.git', 'slirp']
A full log can be found at /root/qemu-8.2.10/build/meson-logs/meson-log.txt
ERROR: meson setup failed
解决方法：
由于无法访问https://gitlab.freedesktop.org/slirp/libslirp.git，将其改成
https://gitlab.com/qemu-project/libslirp.git
vim ./subprojects/slirp.wrap
[wrap-git]
url = https://gitlab.com/qemu-project/libslirp.git
revision = 26be815b86e8d49add8c9a8b320239b9594ff03d
[provide]
slirp = libslirp_dep-
busybox CROSS_COMPILE 编译出错
root@kernel:~/busybox-1.35.0# ARCH=aarch64 make CROSS_COMPILE=aarch64-linux-gnu- install -j16
CC networking/tc.o
CC networking/tftp.o
CC networking/tls.o
CC networking/tls_aes.o
CC networking/tls_fe.o
CC networking/tls_aesgcm.o
CC networking/tls_pstm.o
CC networking/tls_pstm_montgomery_reduce.o
CC networking/tls_pstm_mul_comba.o
CC networking/tls_pstm_sqr_comba.o
CC networking/tls_rsa.o
CC networking/tls_sp_c32.o
CC networking/traceroute.o
CC networking/tunctl.o
networking/tc.c: In function 'cbq_print_opt':
networking/tc.c:236:27: error: 'TCA_CBQ_MAX' undeclared (first use in this function); did you mean 'TCA_CBS_MAX'?
236 | struct rtattr *tb[TCA_CBQ_MAX+1];
| ^~~~~~~~~~~
| TCA_CBS_MAX
networking/tc.c:236:27: note: each undeclared identifier is reported only once for each function it appears in
networking/tc.c:249:16: error: 'TCA_CBQ_RATE' undeclared (first use in this function); did you mean 'TCA_TBF_RATE64'?
249 | if (tb[TCA_CBQ_RATE]) {
| ^~~~~~~~~~~~
| TCA_TBF_RATE64
networking/tc.c:255:16: error: 'TCA_CBQ_LSSOPT' undeclared (first use in this function)
255 | if (tb[TCA_CBQ_LSSOPT]) {
| ^~~~~~~~~~~~~~
networking/tc.c:256:61: error: invalid application of 'sizeof' to incomplete type 'struct tc_cbq_lssopt'
256 | if (RTA_PAYLOAD(tb[TCA_CBQ_LSSOPT]) < sizeof(*lss))
| ^
networking/tftp.c: In function 'tftpd_main':
networking/tftp.c:886:15: warning: 'local_file' is used uninitialized [-Wuninitialized]
886 | char *local_file = local_file;
| ^~~~~~~~~~
networking/tftp.c:886:15: note: 'local_file' was declared here
886 | char *local_file = local_file;
| ^~~~~~~~~~
CC networking/vconfig.o
networking/tc.c:261:16: error: 'TCA_CBQ_WRROPT' undeclared (first use in this function)
261 | if (tb[TCA_CBQ_WRROPT]) {
| ^~~~~~~~~~~~~~
networking/tc.c:262:61: error: invalid application of 'sizeof' to incomplete type 'struct tc_cbq_wrropt'
262 | if (RTA_PAYLOAD(tb[TCA_CBQ_WRROPT]) < sizeof(*wrr))
| ^
networking/tc.c:267:16: error: 'TCA_CBQ_FOPT' undeclared (first use in this function)
267 | if (tb[TCA_CBQ_FOPT]) {
| ^~~~~~~~~~~~
networking/tc.c:268:59: error: invalid application of 'sizeof' to incomplete type 'struct tc_cbq_fopt'
268 | if (RTA_PAYLOAD(tb[TCA_CBQ_FOPT]) < sizeof(*fopt))
| ^
CC networking/wget.o
networking/tc.c:273:16: error: 'TCA_CBQ_OVL_STRATEGY' undeclared (first use in this function)
273 | if (tb[TCA_CBQ_OVL_STRATEGY]) {
| ^~~~~~~~~~~~~~~~~~~~
networking/tc.c:274:67: error: invalid application of 'sizeof' to incomplete type 'struct tc_cbq_ovl'
274 | if (RTA_PAYLOAD(tb[TCA_CBQ_OVL_STRATEGY]) < sizeof(*ovl))
| ^
networking/tc.c:277:50: error: invalid application of 'sizeof' to incomplete type 'struct tc_cbq_ovl'
277 | (unsigned) sizeof(*ovl));
| ^
networking/tc.c:293:23: error: invalid use of undefined type 'struct tc_cbq_lssopt'
293 | if (lss && lss->flags) {
| ^~
networking/tc.c:296:24: error: invalid use of undefined type 'struct tc_cbq_lssopt'
296 | if (lss->flags&TCF_CBQ_LSS_BOUNDED) {
| ^~
CC networking/whois.o
CC networking/zcip.o
networking/tc.c:296:32: error: 'TCF_CBQ_LSS_BOUNDED' undeclared (first use in this function)
296 | if (lss->flags&TCF_CBQ_LSS_BOUNDED) {
| ^~~~~~~~~~~~~~~~~~~
networking/tc.c:300:24: error: invalid use of undefined type 'struct tc_cbq_lssopt'
300 | if (lss->flags&TCF_CBQ_LSS_ISOLATED) {
| ^~
networking/tc.c:300:32: error: 'TCF_CBQ_LSS_ISOLATED' undeclared (first use in this function)
300 | if (lss->flags&TCF_CBQ_LSS_ISOLATED) {
| ^~~~~~~~~~~~~~~~~~~~
networking/tc.c:308:24: error: invalid use of undefined type 'struct tc_cbq_wrropt'
308 | if (wrr->priority != TC_CBQ_MAXPRIO)
| ^~
networking/tc.c:308:38: error: 'TC_CBQ_MAXPRIO' undeclared (first use in this function)
308 | if (wrr->priority != TC_CBQ_MAXPRIO)
| ^~~~~~~~~~~~~~
networking/tc.c:309:46: error: invalid use of undefined type 'struct tc_cbq_wrropt'
309 | printf("prio %u", wrr->priority);
| ^~
networking/tc.c:313:43: error: invalid use of undefined type 'struct tc_cbq_wrropt'
313 | printf("/%u ", wrr->cpriority);
| ^~
networking/tc.c:314:32: error: invalid use of undefined type 'struct tc_cbq_wrropt'
314 | if (wrr->weight != 1) {
| ^~
networking/tc.c:315:65: error: invalid use of undefined type 'struct tc_cbq_wrropt'
315 | print_rate(buf, sizeof(buf), wrr->weight);
| ^~
networking/tc.c:318:32: error: invalid use of undefined type 'struct tc_cbq_wrropt'
318 | if (wrr->allot)
| ^~
networking/tc.c:319:57: error: invalid use of undefined type 'struct tc_cbq_wrropt'
319 | printf("allot %ub ", wrr->allot);
| ^~
networking/tc.c:236:24: warning: unused variable 'tb' [-Wunused-variable]
236 | struct rtattr *tb[TCA_CBQ_MAX+1];
| ^~
make[1]: *** [scripts/Makefile.build:197: networking/tc.o] Error 1
make[1]: *** Waiting for unfinished jobs....
networking/wget.c: In function 'retrieve_file_data':
networking/wget.c:1085:33: warning: ignoring return value of 'ftruncate' declared with attribute 'warn_unused_result' [-Wunused-result]
1085 | ftruncate(G.output_fd, pos);
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~
make: *** [Makefile:744: networking] Error 2
解决方法
修改 BusyBox 的配置文件
make menuconfig取消 tc 这个工具的编译，
Networking Utilities -->
[ ] tc 7. 参考文献
https://blog.csdn.net/nanhai_happy/article/details/124941074?spm=1011.2415.3001.5331
https://zhuanlan.zhihu.com/p/624853021
https://blog.csdn.net/thisinnocence/article/details/127931774
https://blog.csdn.net/nanhai_happy/article/details/146915229?spm=1001.2014.3001.5502
https://blog.csdn.net/nanhai_happy/article/details/124835581