Views
No views yet
0.11.0.dev01adapter: lora
2base_model: unsloth/Llama-3.2-1B-Instruct
3bf16: true
4fp16: false
5load_in_4bit: false
6load_in_8bit: false
7datasets:
8 - path: mx003/viper
9 type: chat_template
10 field_messages: messages
11 chat_template: llama3
12val_set_size: 0.05
13lora_r: 32
14lora_alpha: 64
15lora_dropout: 0.05
16lora_target_modules:
17 - q_proj
18 - v_proj
19 - k_proj
20 - o_proj
21 - gate_proj
22 - down_proj
23 - up_proj
24gradient_accumulation_steps: 8
25gradient_checkpointing: true
26micro_batch_size: 2
27num_epochs: 2
28learning_rate: 0.0002
29optimizer: adamw_torch_fused
30lr_scheduler: cosine
31warmup_ratio: 0.03
32output_dir: ./outputs/mymodel-1b
33sequence_len: 4096
34save_steps: 10
35logging_steps: 5
36flash_attention: false
37sample_packing: true
38group_by_length: false
39train_on_inputs: false
40special_tokens:
41 pad_token: <|end_of_text|>
42
43