安装Conda工具
bash
安装 Miniforge Conda
wget -c https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh
chmod 777 Miniforge3-Linux-x86_64.sh
bash Miniforge3-Linux-x86_64.sh
source ~/miniforge3/bin/activate # Miniforge 安装的目录
# 创建名称为 toolkit2 的 Python 3.8 环境
conda create -n toolkit2 python=3.8
# 激活toolkit2环境
conda activate toolkit2
工程代码下载
bash
# 新建 Projects 文件夹
mkdir Projects
# 进入该目录
cd Projects
# 下载 RKNN-Toolkit2 仓库
git clone https://github.com/airockchip/rknn-toolkit2.git --depth 1
# 下载 RKNN Model Zoo 仓库
git clone https://github.com/airockchip/rknn_model_zoo.git --depth 1
# 下载 rknn-llm 仓库
git clone https://github.com/airockchip/rknn-llm.git --depth 1
确认板端环境
安装依赖环境以及YOLOv5运行
bash
# 安装依赖库和 RKNN-Toolkit2
# 进入 rknn-toolkit2 目录
cd Projects/rknn-toolkit2/rknn-toolkit2
# 请根据不同的 python 版本,选择不同的 requirements 文件
# 例如 python3.8 对应 requirements_cp38.txt
pip install -r packages/requirements_cpxx.txt
# 安装 RKNN-Toolkit2
# 请根据不同的 python 版本及处理器架构,选择不同的 wheel 安装包文件:
# 其中 x.x.x 是 RKNN-Toolkit2 版本号,cpxx 是 python 版本号,请根据实际数值进行替换
pip install packages/rknn_toolkit2--x.x.x-cpxx-cpxx-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
# 验证是否安装成功
# 进入 Python 交互模式
python
# 导入 RKNN 类
from rknn.api import RKNN
# 更新包列表
sudo apt update
# 安装 cmake
sudo apt install cmake
# 进入 rknpu2 目录
cd Projects/rknn-toolkit2/rknpu2
# 推送 rknn_server 到板端
# 注:在64位Linux系统中,BOARD_ARCH对应a
# adb push runtime/Linux/rknn_server/${BOARD_ARC
adb push runtime\Linux\rknn_server\aarch64\usr\bin\rknn_server /usr/bin
adb push runtime\Linux\rknn_server\aarch64\usr\bin\start_rknn.sh /usr/bin
adb push runtime\Linux\rknn_server\aarch64\usr\bin\restart_r knn.sh /usr/bin
# 推送 librknnrt.so
# adb push runtime/Linux/librknn_api/${BOARD_ARC
adb push runtime\Linux\rlibrknn_api\aarch64\librknnrt.so / usr/lib
# 进入板端
adb shell
# 赋予可执行权限
chmod +x /usr/bin/rknn_server
chmod +x /usr/bin/start_rknn.sh
chmod +x /usr/bin/restart_rknn.sh
# 重启 rknn_server 服务
restart_rknn.sh
RKNN C Demo 使用方法
准备模型
# 进入 rknn_model_zoo/examples/yolov5/model 目录
cd Projects/rknn_model_zoo/examples/yolov5/model
# 运行 download_model.sh 脚本,下载 yolov5 onnx 模型
# 例如,下载好的 onnx 模型存放路径为 model/yolov5s_relu.onnx
./download_model.sh
模型转换
# 进入 rknn_model_zoo/examples/yolov5/python 目录
cd Projects/rknn_model_zoo/examples/yolov5/python
# 运行 convert.py 脚本,将原始的 ONNX 模型转成 RKNN 模型
# 用法: python convert.py model_path [rk3566|rk3588|rk3562] [i8/fp] [output_path]
python convert.py ../model/yolov5s_relu.onnx rk3576 i8 ../model/yolov5s_relu.rknn
# 添加到 build-linux.sh 脚本的开头位置即可
GCC_COMPILER=Projects/gcc-linaro-6.3.1-2017.05-x86_64_aarch64-linux-gnu/bin/aarch64-linux-gnu
# 用的是sdk自带的gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu
# 路径 linux/prebuilts/gcc/linux-x86/aarch64/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu
--- a/build-linux.sh
+++ b/build-linux.sh
@@ -1,5 +1,7 @@
#!/bin/bash
+GCC_COMPILER=Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-rockchip1031-linux-gnu
+
set -e
echo "$0 $@"
# 编译之前修改 库文件
cd 3rdparty/mpp/Linux/aarch64/
git status ./
rm librockchip_mpp.so
mv librockchip_mpp.so.0 librockchip_mpp.so
cd ..
cd rknn_yolov5_demo/
# 编译
./build-linux.sh -t rk3576 -a aarch64 -b Release
(toolkit2) ip3@ip3-MOMENTPLUS-M2:~/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo$ ./build-linux.sh -t rk3576 -a aarch64 -b Release
./build-linux.sh -t rk3576 -a aarch64 -b Release
/home/ip3/work/linux/Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-rockchip1031-linux-gnu
===================================
TARGET_SOC=RK3576
TARGET_ARCH=aarch64
BUILD_TYPE=Release
BUILD_DIR=/home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/build/build_RK3576_linux_aarch64_Release
CC=/home/ip3/work/linux/Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-rockchip1031-linux-gnu-gcc
CXX=/home/ip3/work/linux/Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-rockchip1031-linux-gnu-g++
===================================
-- Configuring done
-- Generating done
-- Build files have been written to: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/build/build_RK3576_linux_aarch64_Release
Consolidate compiler generated dependencies of target rknn_yolov5_video_demo
Consolidate compiler generated dependencies of target rknn_yolov5_demo
[ 60%] Built target rknn_yolov5_video_demo
[100%] Built target rknn_yolov5_demo
[ 40%] Built target rknn_yolov5_demo
[100%] Built target rknn_yolov5_video_demo
Install the project...
-- Install configuration: "Release"
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/./rknn_yolov5_demo
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/lib/librknnrt.so
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/lib/librga.so
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/./model/RK3576
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/./model/RK3576/yolov5s-640-640.rknn
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/./model/bus.jpg
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/./model/coco_80_labels_list.txt
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/./rknn_yolov5_video_demo
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/lib/librockchip_mpp.so
-- Up-to-date: /home/ip3/work/linux/Projects/rknn-toolkit2/rknpu2/examples/rknn_yolov5_demo/install/rknn_yolov5_demo_Linux/lib/libmk_api.so
# 进入 rknn_model_zoo 目录
cd Projects/rknn_model_zoo
# 运行 build-linux.sh 脚本
# 用法:./build-linux.sh -t <target> -a <arch> -d <build_demo_name> [-b <build_type>] [-m]
# -t : target (rk356x/rk3588) # 平台类型,rk3568/rk3566 都统一为rk356x
# -a : arch (aarch64/armhf) # 板端系统架构
# -d : demo name # 对应 examples 目录下子文件夹的名称,如yolov5、mobilenet
# -b : build_type(Debug/Release)
# -m : enable address sanitizer, build_type need set to Debug
chmod +x build-linux.sh
./build-linux.sh -t rk3576 -a aarch64 -d yolov5
(toolkit2) ip3@ip3-MOMENTPLUS-M2:~/work/linux/Projects/rknn_model_zoo$ ./build-linux.sh -t rk3576 -a aarch64 -d yolov5
./build-linux.sh -t rk3576 -a aarch64 -d yolov5
/home/ip3/work/linux/Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-rockchip1031-linux-gnu
===================================
BUILD_DEMO_NAME=yolov5
BUILD_DEMO_PATH=examples/yolov5/cpp
TARGET_SOC=rk3576
TARGET_ARCH=aarch64
BUILD_TYPE=Release
ENABLE_ASAN=OFF
DISABLE_RGA=OFF
INSTALL_DIR=/home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo
BUILD_DIR=/home/ip3/work/linux/Projects/rknn_model_zoo/build/build_rknn_yolov5_demo_rk3576_linux_aarch64_Release
CC=/home/ip3/work/linux/Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-rockchip1031-linux-gnu-gcc
CXX=/home/ip3/work/linux/Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-rockchip1031-linux-gnu-g++
===================================
-- Configuring done
-- Generating done
-- Build files have been written to: /home/ip3/work/linux/Projects/rknn_model_zoo/build/build_rknn_yolov5_demo_rk3576_linux_aarch64_Release
Consolidate compiler generated dependencies of target imageutils
Consolidate compiler generated dependencies of target fileutils
Consolidate compiler generated dependencies of target imagedrawing
Consolidate compiler generated dependencies of target audioutils
[ 25%] Built target imagedrawing
[ 41%] Built target fileutils
[ 50%] Built target audioutils
[ 66%] Built target imageutils
Consolidate compiler generated dependencies of target rknn_yolov5_demo
[100%] Built target rknn_yolov5_demo
[ 16%] Built target imageutils
[ 33%] Built target fileutils
[ 50%] Built target imagedrawing
[ 83%] Built target rknn_yolov5_demo
[100%] Built target audioutils
Install the project...
-- Install configuration: "Release"
-- Installing: /home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo/./rknn_yolov5_demo
-- Set runtime path of "/home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo/./rknn_yolov5_demo" to "$ORIGIN/lib"
-- Installing: /home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo/./model/bus.jpg
-- Installing: /home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo/./model/coco_80_labels_list.txt
-- Installing: /home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo/model/yolov5s_relu.rknn
-- Installing: /home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo/lib/librknnrt.so
-- Installing: /home/ip3/work/linux/Projects/rknn_model_zoo/install/rk3576_linux_aarch64/rknn_yolov5_demo/lib/librga.so
#编译之后的文件
(toolkit2) ip3@ip3-MOMENTPLUS-M2:~/work/linux/Projects/rknn_model_zoo/install$ tree
.
└── rk3576_linux_aarch64
└── rknn_yolov5_demo
├── lib
│ ├── librga.so
│ └── librknnrt.so
├── model
│ ├── bus.jpg
│ ├── coco_80_labels_list.txt
│ └── yolov5s_relu.rknn
└── rknn_yolov5_demo
# 推送文件到板端
# 进入 rknn_model_zoo 目录
cd Projects/rknn_model_zoo
# 推送整个 rknn_yolov5_demo 文件夹到板端
# 注:rknn_yolov5_demo 文件夹下有一个同名的可执行文件 rknn_yolov5_demo
# 注:使用不同的模型和平台时,建议直接在 install 下找对应的路径
adb push install/rk3576_linux_aarch64/rknn_yolov5_demo /data/
# 板端运行 Demo
# 进入板端
adb shell
# 进入 rknn_yolov5_demo 目录
cd /data/rknn_yolov5_demo/
# 设置依赖库环境
export LD_LIBRARY_PATH=./lib
# 运行可执行文件
# 用法: ./rknn_yolov5_demo <model_path> <input_path>
./rknn_yolov5_demo model/yolov5s_relu.rknn model/bus.jpg
# 查看结果
# 拉取到本地当前目录
adb pull /data/rknn_yolov5_demo/out.png .
RKLLM部署
bash
# 代码 rknn-llm
# 环境 激活toolkit2环境
conda activate toolkit2
# 切换到rkllm-toolkit目录
cd rknn-llm/rkllm-toolkit/
# 安装rkllm_toolkit(文件请根据具体版本修改),会自动下载RKLLM-Toolkit工具所需要的相关依赖包。
pip install packages/rkllm_toolkit-1.0.1-cp38-cp38-linux_x86_64.whl
#模型下载
cd Projects
mkdir modem
Qwen-1_8B-Chat_downloader.py
from modelscope import snapshot_download
#model_dir = snapshot_download('qwen/Qwen-7B-Chat',cache_dir='./modem')
model_dir = snapshot_download('qwen/Qwen-1_8B-Chat',cache_dir='./modem')
运行Qwen-1_8B-Chat_downloader.py
python Qwen-1_8B-Chat_downloader.py
# 模型转换
# 在rknn-llm/rkllm-toolkit/examples/下新建 huggingface目录
# 新建download.py 代码如下:
from rkllm.api import RKLLM
# 模型路径
modelpath = '/home/ip3/work/linux/Projects/modem/qwen/Qwen-1_8B-Chat'
# 初始化RKLLM对象
llm = RKLLM()
# 模型加载
ret = llm.load_huggingface(model = modelpath)
if ret != 0:
print('Load model failed!')
exit(ret)
# 模型的量化构建
ret = llm.build(do_quantization=True, optimization_level=1, quantized_dtype='w8a8', target_platform='rk3576',num_npu_core=2)
if ret != 0:
print('Build model failed!')
exit(ret)
# 导出rkllm模型
ret = llm.export_rkllm("./qwen.rkllm")
if ret != 0:
print('Export model failed!')
exit(ret)
# RKLLM Runtime 编译
cd Projects/rknn-llm/rkllm-runtime/examples/rkllm_api_demo
#设置编译工具连接
+++ b/rkllm-runtime/examples/rkllm_api_demo/build-linux.sh
@@ -1,10 +1,12 @@
#!/bin/bash
+
# Debug / Release / RelWithDebInfo
if [[ -z ${BUILD_TYPE} ]];then
BUILD_TYPE=Release
fi
-GCC_COMPILER_PATH=~/gcc-arm-10.2-2020.11-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu
+GCC_COMPILER_PATH=~/work/linux/Projects/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu/bin/aarch64-none-linux-gnu
(toolkit2) ip3@ip3-MOMENTPLUS-M2:~/work/linux/Projects/rknn-llm/rkllm-runtime/examples/rkllm_api_demo$ ./build-linux.sh
-- Configuring done
-- Generating done
-- Build files have been written to: /home/ip3/work/linux/Projects/rknn-llm/rkllm-runtime/examples/rkllm_api_demo/build/build_linux_aarch64_Release
Consolidate compiler generated dependencies of target llm_demo
Consolidate compiler generated dependencies of target multimodel_demo
[ 50%] Built target llm_demo
[100%] Built target multimodel_demo
(toolkit2) ip3@ip3-MOMENTPLUS-M2:~/work/linux/Projects/rknn-llm/rkllm-runtime/examples/rkllm_api_demo/build$ tree -L 2
.
└── build_linux_aarch64_Release
├── CMakeCache.txt
├── CMakeFiles
├── cmake_install.cmake
├── llm_demo
├── Makefile
└── multimodel_demo
2 directories, 5 files
# 推送板端
# 注意:要去到llm_demo 和 librkllmrt.so所在的目录下去执行
adb push llm_demo /data # 将llm_demo使用ssh传输到开发板的家目录
adb push ./rkllm-runtime/runtime/Linux/librkllm_api/aarch64/librkllmrt.so /data/lib
adb push Projects/rknn-llm/rkllm-toolkit/examples/huggingface/qwen.rkllm /data
运行
adb shell && cd data
cd librkllmrt.so所在的目录
export LD_LIBRARY_PATH=./lib
# 修改最大可打开文件的数量
ulimit -HSn 10240
root@linaro-alip:/data# ./llm_demo qwen.rkllm 128 4080
rkllm init start
I rkllm: rkllm-runtime version: 1.1.2, rknpu driver version: 0.9.8, platform: RK3576
# 或者运行绑定cpu运行测试
# taskset f0 ./llm_demo /path/qwen.rkllm