复制代码
# env
apt-get update
apt-get install git-lfs
git init
git lfs install
mkdir Qwen
cd Qwen
git clone https://huggingface.co/Qwen/Qwen-7B
cd ..
git clone https://gitee.com/meijunhui0917/LLaMA-Efficient-Tuning.git
git clone "huanhuan": {
"file_name": "huanhuan.json",
"columns": {
"prompt": "instruction",
"query": "input",
"response": "output"
}
},
复制代码
CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \
--stage sft \
--model_name_or_path ../Qwen/Qwen-7B\
--do_train \
--dataset huanhuan \
--template default \
--finetuning_type lora \
--lora_target c_attn \
--output_dir ./model \
--overwrite_cache \
--per_device_train_batch_size 4 \
--gradient_accumulation_steps 4 \
--lr_scheduler_type cosine \
--logging_steps 10 \
--save_steps 1000 \
--learning_rate 5e-5 \
--num_train_epochs 3.0 \
--plot_loss \
--fp16
python
复制代码
CUDA_VISIBLE_DEVICES=0 python src/train_bash.py \
--stage sft \
--model_name_or_path ../Qwen/Qwen-7B \
--do_train True \
--overwrite_cache True \
--finetuning_type lora \
--template chatml \
--dataset huanhuan \
--max_source_length 512 \
--max_target_length 512 \
--learning_rate 5e-05 \
--num_train_epochs 3.0 \
--max_samples 100000 \
--per_device_train_batch_size 1 \
--gradient_accumulation_steps 4 \
--lr_scheduler_type cosine \
--max_grad_norm 1.0 \
--logging_steps 5 \
--save_steps 100 \
--warmup_steps 0 \
--padding_side left \
--lora_rank 8 \
--lora_dropout 0.1 \
--lora_target c_attn \
--resume_lora_training True \
--output_dir saves/Qwen-7B-chat/lora/2023-08-22-17-23-51 \
--fp16 True \
--plot_loss True
复制代码
复制代码
复制代码
复制代码
复制代码
复制代码