Views
No views yet
NewEden/Orion-LITUnsloth/phi-41base_model: unsloth_phi-4
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5 #hub_model_id: NewEden/Phi4-pretrain
6 #hub_strategy: "all_checkpoints"
7 #push_dataset_to_hub:
8 #hf_use_auth_token: true
9
10plugins:
11 - axolotl.integrations.liger.LigerPlugin
12liger_rope: true
13liger_rms_norm: true
14liger_swiglu: true
15liger_fused_linear_cross_entropy: true
16
17 #plugins:
18 # - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
19
20 #cut_cross_entropy: true
21
22load_in_8bit: false
23load_in_4bit: false
24strict: false
25
26datasets:
27 - path: Mielikki/Erebus-87k
28 type: completion
29 field: body
30 - path: NewEden/Orion-Asstr-Stories-16K
31 type: completion
32 field: content
33shuffle_merged_datasets: true
34dataset_prepared_path: prepared_data
35val_set_size: 0.0
36output_dir: ./phi4-pt-out-r2
37
38sequence_len: 16384
39sample_packing: true
40pad_to_sequence_len: true
41
42adapter: lora
43lora_model_dir:
44lora_r: 128
45lora_alpha: 16
46lora_dropout: 0.05
47lora_target_modules:
48 - gate_proj
49 - down_proj
50 - up_proj
51 - q_proj
52 - v_proj
53 - k_proj
54 - o_proj
55
56lora_modules_to_save:
57 - embed_tokens
58 - lm_head
59
60
61wandb_project: mag-phi
62wandb_entity:
63wandb_watch:
64wandb_name: attempt-02
65wandb_log_model:
66
67gradient_accumulation_steps: 4
68micro_batch_size: 2
69num_epochs: 1
70optimizer: paged_ademamix_8bit
71lr_scheduler: cosine
72learning_rate: 0.00001
73
74train_on_inputs: false
75group_by_length: false
76bf16: auto
77fp16:
78tf32: false
79
80gradient_checkpointing: unsloth
81early_stopping_patience:
82resume_from_checkpoint:
83local_rank:
84logging_steps: 1
85xformers_attention:
86flash_attention: true
87
88warmup_steps: 10
89evals_per_epoch: 4
90eval_table_size:
91eval_max_new_tokens: 128
92saves_per_epoch: 4
93debug:
94deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16_cpuoffload_params.json
95weight_decay: 0.01
96fsdp:
97fsdp_config:
