Views
No views yet
1 # LoRA attention dimension
2lora_r = 64
3
4# Alpha parameter for LoRA scaling
5lora_alpha = 16
6
7# Dropout probability for LoRA layers
8lora_dropout = 0.1
9
10################################################################################
11# bitsandbytes parameters
12################################################################################
13
14# Activate 4-bit precision base model loading
15use_4bit = True
16
17# Compute dtype for 4-bit base models
18bnb_4bit_compute_dtype = "float16"
19
20# Quantization type (fp4 or nf4)
21bnb_4bit_quant_type = "nf4"
22
23# Activate nested quantization for 4-bit base models (double quantization)
24use_nested_quant = False
25
26################################################################################
27# TrainingArguments parameters
28################################################################################
29
30# Output directory where the model predictions and checkpoints will be stored
31output_dir = "./results"
32
33# Number of training epochs
34num_train_epochs = 1
35
36# Enable fp16/bf16 training (set bf16 to True with an A100)
37fp16 = False
38bf16 = True
39
40# Batch size per GPU for training
41per_device_train_batch_size = 2
42
43# Batch size per GPU for evaluation
44per_device_eval_batch_size = 2
45
46# Number of update steps to accumulate the gradients for
47gradient_accumulation_steps = 1
48
49# Enable gradient checkpointing
50gradient_checkpointing = True
51
52# Maximum gradient normal (gradient clipping)
53max_grad_norm = 0.3
54
55# Initial learning rate (AdamW optimizer)
56learning_rate = 2e-4
57
58# Weight decay to apply to all layers except bias/LayerNorm weights
59weight_decay = 0.001
60
61# Optimizer to use
62optim = "paged_adamw_32bit"
63
64# Learning rate schedule (constant a bit better than cosine)
65lr_scheduler_type = "constant"
66
67# Number of training steps (overrides num_train_epochs)
68max_steps = -1
69
70# Ratio of steps for a linear warmup (from 0 to learning rate)
71warmup_ratio = 0.03
72
73# Group sequences into batches with same length
74# Saves memory and speeds up training considerably
75group_by_length = True
76
77# Save checkpoint every X updates steps
78save_steps = 25
79
80# Log every X updates steps
81logging_steps = 25
82
83################################################################################
84# SFT parameters
85################################################################################
86
87# Maximum sequence length to use
88max_seq_length = None
89
90# Pack multiple short examples in the same input sequence to increase efficiency
91packing = False
92
93# Load the entire model on the GPU 0
94device_map = {"": 0}