1base_model: IntervitensInc/Llama-3.1-Minitron-4B-Width-Base-chatml
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9datasets:
10 - path: MangoHQ/Gryphe-3.5-16k-Subset
11 type: sharegpt
12 conversation: chatml
13 - path: Epiculous/Synthstruct-Gens-v1-Filtered-n-Cleaned
14 type: sharegpt
15 conversation: chatml
16 - path: anthracite-org/Stheno-Data-Filtered
17 type: sharegpt
18 conversation: chatml
19 - path: Epiculous/SynthRP-Gens-v1-Filtered-n-Cleaned
20 type: sharegpt
21 conversation: chatml
22 - path: lodrick-the-lafted/NopmWritingStruct
23 type: sharegpt
24 conversation: chatml
25 - path: anthracite-org/kalo-opus-instruct-22k-no-refusal
26 type: sharegpt
27 conversation: chatml
28
29chat_template: chatml
30
31val_set_size: 0.01
32output_dir: ./outputs/out
33
34adapter:
35lora_r:
36lora_alpha:
37lora_dropout:
38lora_target_linear:
39
40sequence_len: 16384
41# sequence_len: 32768
42sample_packing: true
43eval_sample_packing: false
44pad_to_sequence_len: true
45
46wandb_project: tinymagnumv2
47wandb_entity:
48wandb_watch:
49wandb_name: tinymagnumv2
50wandb_log_model:
51
52gradient_accumulation_steps: 32
53micro_batch_size: 1
54num_epochs: 2
55optimizer: adamw_bnb_8bit
56lr_scheduler: cosine
57learning_rate: 0.00002
58weight_decay: 0.05
59
60train_on_inputs: false
61group_by_length: false
62bf16: auto
63fp16:
64tf32: true
65
66gradient_checkpointing: true
67early_stopping_patience:
68resume_from_checkpoint:
69local_rank:
70logging_steps: 1
71xformers_attention:
72flash_attention: true
73
74warmup_ratio: 0.1
75evals_per_epoch: 4
76eval_table_size:
77eval_max_new_tokens: 128
78saves_per_epoch: 1
79
80debug:
81deepspeed:
82fsdp:
83fsdp_config:
84
85special_tokens:
86 pad_token: <|finetune_right_pad_id|>
87
