rdma-core debug

export MLX5_DEBUG_MASK=0xff

export MLX5_DEBUG_FILE=/tmp/mlx5.txt

git clone https://github.com/linux-rdma/rdma-core.git

cd rdma-core

./build.sh

修改build/CMakeCache.txt

MLX5_DEBUG:BOOL=TRUE

复制代码
function install_rdma_core
{
        local dir=/swgwork/cmi/rdma-core/build/lib

        cd /lib64/

        /bin/cp $dir/librdmacm.so.1.3.56.0      .
        /bin/cp $dir/libibverbs.so.1.14.56.0    .
        /bin/cp $dir/libmlx5.so.1.25.56.0       .

        /bin/rm librdmacm.so.1
        /bin/rm librdmacm.so
        /bin/rm libibverbs.so.1
        /bin/rm libibverbs.so
        /bin/rm libibverbs/libmlx5-rdmav34.so
        /bin/rm libmlx5.so.1

        ln -s librdmacm.so.1.3.56.0 librdmacm.so
        ln -s librdmacm.so.1.3.56.0 librdmacm.so.1
        ln -s libibverbs.so.1.14.56.0 libibverbs.so
        ln -s libibverbs.so.1.14.56.0 libibverbs.so.1
        ln -s libmlx5.so.1.25.56.0 libmlx5.so.1

        cd libibverbs
        ln -s ../libmlx5.so.1.25.56.0 libmlx5-rdmav34.so
}

运行ib_send_bw:

ip netns exec n11 /swgwork/cmi/perftest/ib_send_bw -d mlx5_3 -z

ip netns exec n11 /swgwork/cmi/perftest/ib_send_bw -d mlx5_3 1.1.1.1 -D 10000 -z

server端收包的时候得到下面的log:

tail -f /tmp/mlx5.txt

mlx5_get_next_cqe:564: dump cqe for cqn 0x1a9e:

00000000 00000000 00000000 00000000

00000000 00000000 00000000 00000000

00000000 00000000 00000000 00010000

0001d00e 98d17383 0000024d 03e3f720

mlx5_get_next_cqe:564: dump cqe for cqn 0x1a9e:

00000000 00000000 00000000 00000000

00000000 00000000 00000000 00000000

00000000 00000000 00000000 00010000

0001d00e 98d17f13 0000024d 03e46b20

mlx5_get_next_cqe:564: dump cqe for cqn 0x1a9e:

00000000 00000000 00000000 00000000

00000000 00000000 00000000 00000000

00000000 00000000 00000000 00010000

0001d00e 98d18aa3 0000024d 03e52e20

上面的输出对应下面的数据结构:

复制代码
struct mlx5_cqe64 {
        union {
                struct {
                        uint8_t         rsvd0[2];
                        __be16          wqe_id;
                        uint8_t         rsvd4[13];
                        uint8_t         ml_path;
                        uint8_t         rsvd20[4];
                        __be16          slid;
                        __be32          flags_rqpn;
                        uint8_t         hds_ip_ext;
                        uint8_t         l4_hdr_type_etc;
                        __be16          vlan_info;
                };
                struct mlx5_tm_cqe tm_cqe;
                /* TMH is scattered to CQE upon match */
                struct ibv_tmh tmh;
        };
        __be32          srqn_uidx;
        __be32          imm_inval_pkey;
        uint8_t         app;
        uint8_t         app_op;
        __be16          app_info;
        __be32          byte_cnt;
        __be64          timestamp;
        __be32          sop_drop_qpn;
        __be16          wqe_counter;
        uint8_t         signature;
        uint8_t         op_own;
};

比如上面的例子byte_cnt就是0x10000(65536),wqe_counter分别是03e4,03e5,timestamp分别是0001d00e 98d17f13,0001d00e 98d18aa3。

crash> ps | grep ib_send_bw

> 685270 2581 23 ffff90c5b0c28000 RU 0.0 6716 4832 ib_send_bw

crash> files ffff90c5b0c28000

PID: 685270 TASK: ffff90c5b0c28000 CPU: 23 COMMAND: "ib_send_bw"

ROOT: / CWD: /labhome/cmi/mi/drgn/ib

FD FILE DENTRY INODE TYPE PATH

0 ffff90c3cda4eb80 ffff90c3cc6399a0 ffff90c3d1325a40 CHR /dev/pts/9

1 ffff90c3cda4eb80 ffff90c3cc6399a0 ffff90c3d1325a40 CHR /dev/pts/9

2 ffff90c3cda4eb80 ffff90c3cc6399a0 ffff90c3d1325a40 CHR /dev/pts/9

3 ffff90c4de695200 ffff90c5504eccd0 ffff90c3cf2dd978 CHR /dev/infiniband/uverbs3

4 ffff90c4de695980 ffff90c4c814c148 ffff90c4f9ca6380 REG /tmp/mlx5.txt

5 ffff90c4de695f80 ffff90c4c81c8000 ffff90c3c05ed580 UNKN [infinibandevent]

6 ffff90c4de694c00 ffff90c47ffcf710 ffff90c3e222a0f0 CHR /dev/infiniband/rdma_cm

7 ffff90c4de697780 ffff90c5504eccd0 ffff90c3cf2dd978 CHR /dev/infiniband/uverbs3

8 ffff90c4de695b00 ffff90c4c814c148 ffff90c4f9ca6380 REG /tmp/mlx5.txt

9 ffff90c4de696e80 ffff90c4c81c83d8 ffff90c3c05ed580 UNKN [infinibandevent]

相关推荐
禁默25 分钟前
已知 inode 号,如何操作文件?Ext 文件系统增删查改底层逻辑拆解
linux·服务器·数据库
许怀楠31 分钟前
【Linux】基础指令(下)
linux
咕咚.萌西1 小时前
RISC-V开发环境搭建
linux·硬件架构·risc-v
NiKo_W2 小时前
Linux 初识
linux·运维·服务器
磊灬泽6 小时前
【日常错误】鼠标无反应
linux·windows
Miracle&10 小时前
2.TCP深度解析:握手、挥手、状态机、流量与拥塞控制
linux·网络·tcp/ip
专注API从业者10 小时前
Python/Java 代码示例:手把手教程调用 1688 API 获取商品详情实时数据
java·linux·数据库·python
Ribou11 小时前
Ubuntu 24.04.2安装k8s 1.33.4 配置cilium
linux·ubuntu·kubernetes
tan180°12 小时前
Boost搜索引擎 网络库与前端(4)
linux·网络·c++·搜索引擎
Mr. Cao code13 小时前
Docker:颠覆传统虚拟化的轻量级革命
linux·运维·ubuntu·docker·容器