1base_model: arcee-ai/Llama-3.1-SuperNova-Lite
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9datasets:
10 - path: NewEden/CharacterAI-logs-sharegpt-Ngram-Cleaned
11 type: sharegpt
12 conversation: llama3
13 - path: NewEden/OpenCAI-ShareGPT
14 type: sharegpt
15 conversation: llama3
16
17
18chat_template: llama3
19
20 #val_set_size: 0.01
21output_dir: ./outputs
22
23adapter:
24lora_r:
25lora_alpha:
26lora_dropout:
27lora_target_linear:
28
29sequence_len: 16384
30# sequence_len: 32768
31sample_packing: true
32eval_sample_packing: false
33pad_to_sequence_len: true
34
35
36wandb_project: CAI-Supernova
37wandb_entity:
38wandb_watch:
39wandb_name: CAI-Supernova-1
40wandb_log_model:
41
42
43plugins:
44 - axolotl.integrations.liger.LigerPlugin
45liger_rope: true
46liger_rms_norm: true
47liger_swiglu: true
48liger_fused_linear_cross_entropy: true
49
50gradient_accumulation_steps: 4
51micro_batch_size: 1
52num_epochs: 4
53optimizer: paged_adamw_8bit
54lr_scheduler: cosine
55learning_rate: 2e-6
56weight_decay: 0.05
57
58train_on_inputs: false
59group_by_length: false
60bf16: auto
61fp16:
62tf32: true
63
64gradient_checkpointing: unsloth
65early_stopping_patience:
66resume_from_checkpoint:
67#auto_resume_from_checkpoints: true
68local_rank:
69logging_steps: 1
70xformers_attention:
71flash_attention: true
72
73warmup_steps: 5
74 #evals_per_epoch: 4
75eval_table_size:
76 #eval_max_new_tokens: 128
77saves_per_epoch: 1
78
79debug:
80deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16_cpuoffload_params.json
81fsdp:
82fsdp_config:
83
84special_tokens:
85 pad_token: <|finetune_right_pad_id|>
86 eos_token: <|eot_id|>
87
88
