下载镜像
docker pill http://nvcr.io/nvidia/tritonserver:22.07-py3
docker run --gpus all -itd -p8000:8000 -p8001:8001 -p8002:8002 -v /home/ai-developer/server/docs/examples/model_repository/:/models nvcr.io/nvidia/tritonserver:22.07-py3
docker exec -it a5bc bash
tritonserver --model-repository=/models --strict-model-config=false
非必要config文件
支持的格式有TrnsorRT,TensorFLOW saved-model,ONNX models do not require config.pbtxt when --strict-model-config=false
config中,platfrom可填写Tensorrt_plan,onnxruntime_onnx,pytorch_libtorch
backend tensorrt,onnxruntime,pytorch
dims: [ 3,-1,-1 ] -1代表可变维度
--model-control-model explicit
git clone https://github.com/NVIDIA/DeepLearningExamples.git
cd data/squad/
下载数据集
sh squad_download.sh
模型映射文件地址
cd /models
下载演示模型的地址
https://catalog.ngc.nvidia.com/orgs/nvidia/models/bert_pyt_ckpt_large_qa_squad11_amp
#粘贴wget命令
wget --content-disposition https://api.ngc.nvidia.com/v2/models/nvidia/bert_pyt_ckpt_large_qa_squad11_amp/versions/19.09.0/zip -O bert_pyt_ckpt_large_qa_squad11_amp_19.09.0.zip
转换格式
python3 triton/export_model.py
--input-path triton/model.py
--input-type pyt
--output-path $/models/exported_model.onnx
--output-type onnx
--dataloader triton/dataloader.py
--ignore-unknown-parameters
--onnx-opset 13
${FLAG}
--config-file bert_configs/large.json
--checkpoint /models/bert_large_qa.pt
--precision fp16
--vocab-file /models/vocab.txt
--max-seq-length 34
--predict-file /opt/tritonserver/DeepLearningExamples/PyTorch/LanguageModeling/BERT/data/squad/v1.1/dev-v1.1.json
--batch-size 16