File size: 1,895 Bytes
fc4978b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
#!/bin/bash
#source /media/nmitchko/NVME/text-generation-ui/venv/bin/activate
source /media/nmitchko/NVME/text-generation-webui/venv/bin/activate
CURRENTDATEONLY=`date +"%b %d %Y"`
# Change Power limit to 190 for training
sudo nvidia-smi -i 1 -pl 250
sudo nvidia-smi -i 0 -pl 250
export CUDA_VISIBLE_DEVICES=0,1
accelerate launch --num_processes 2 qlora.py \
--ddp_find_unused_parameters False \
--model_name_or_path /media/nmitchko/NVME/text-generation-webui/models/codellama_CodeLlama-34b-hf \
--output_dir /media/ai/blk/loras/i2b2training \
--logging_steps 100 \
--save_strategy steps \
--data_seed 42 \
--save_steps 200 \
--save_total_limit 40 \
--evaluation_strategy steps \
--eval_dataset_size 1024 \
--max_eval_samples 1000 \
--per_device_eval_batch_size 2 \
--per_device_train_batch_size 2 \
--trust_remote_code True \
--use_auth_token False \
--max_new_tokens 32 \
--dataloader_num_workers 2 \
--group_by_length \
--logging_strategy steps \
--remove_unused_columns False \
--do_train \
--lora_r 64 \
--lora_alpha 16 \
--lora_modules all \
--double_quant \
--quant_type nf4 \
--bf16 \
--bits 4 \
--legacy=False \
--warmup_ratio 0.03 \
--lr_scheduler_type constant \
--gradient_checkpointing \
--dataset="i2b2.json" \
--dataset_format alpaca \
--trust_remote_code=True \
--source_max_len 16 \
--target_max_len 512 \
--per_device_train_batch_size 2 \
--gradient_accumulation_steps 16 \
--max_steps 4500 \
--eval_steps 1000 \
--learning_rate 0.0001 \
--adam_beta2 0.999 \
--max_grad_norm 0.3 \
--lora_dropout 0.05 \
--weight_decay 0.0 \
--seed 0 > "${CURRENTDATEONLY}-finetune-i2b2.log" &
# Change Power limit to 300 for normal activities training
# Not Needed for non-managed script
deactivate
|