Views
No views yet
<|begin_of_text|><|start_header_id|>system<|end_header_id|>
This is text
{system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>
{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>1base_model: alpindale/Llama-3.2-3B-Instruct
2model_type: LlamaForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: true
7strict: false
8
9chat_template: llama3
10datasets:
11 - path: Fischerboot/small-boi-thinkin
12 type: sharegpt
13 conversation: llama3
14dataset_prepared_path: last_run_prepared
15val_set_size: 0.1
16output_dir: ./outputs/yuh
17
18adapter: qlora
19lora_model_dir:
20
21sequence_len: 4096
22sample_packing: false
23pad_to_sequence_len: true
24
25lora_r: 32
26lora_alpha: 16
27lora_dropout: 0.05
28lora_target_linear: true
29lora_fan_in_fan_out:
30lora_target_modules:
31 - gate_proj
32 - down_proj
33 - up_proj
34 - q_proj
35 - v_proj
36 - k_proj
37 - o_proj
38
39wandb_project:
40wandb_entity:
41wandb_watch:
42wandb_name:
43wandb_log_model:
44
45gradient_accumulation_steps: 1
46micro_batch_size: 1
47num_epochs: 1
48optimizer: adamw_bnb_8bit
49lr_scheduler: cosine
50learning_rate: 0.0002
51
52train_on_inputs: false
53group_by_length: false
54bf16: auto
55fp16:
56tf32: false
57
58gradient_checkpointing: true
59early_stopping_patience:
60resume_from_checkpoint:
61local_rank:
62logging_steps: 1
63xformers_attention:
64flash_attention: true
65
66loss_watchdog_threshold: 8.0
67loss_watchdog_patience: 3
68
69eval_sample_packing: false
70warmup_steps: 10
71evals_per_epoch: 2
72eval_table_size:
73eval_max_new_tokens: 128
74saves_per_epoch: 2
75debug:
76deepspeed:
77weight_decay: 0.0
78fsdp:
79fsdp_config:
80special_tokens:
81 bos_token: "<|begin_of_text|>"
82 eos_token: "<|end_of_text|>"
83 pad_token: "<|end_of_text|>"| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.5032 | 0.0000 | 1 | 1.6556 |
| 1.2011 | 0.5000 | 10553 | 0.6682 |