Views
No views yet
0.9.21base_model: giux78/zagreus-test-202000
2# Automatically upload checkpoint and final model to HF
3# hub_model_id: username/custom_model_name
4
5#tokenizer_type: auto
6strict: false
7seed: 1337
8output_dir: ./ale_outputs/zagreus-350M-sft
9
10# === Datasets ===
11streaming: false
12datasets:
13 - path: /leonardo_work/EUHPC_A04_045/training/sft_data #/leonardo_work/EUHPC_A04_045/training/test_data #/leonardo_work/EUHPC_A04_045/.data
14 type: chat_template
15 chat_template: tokenizer_default_fallback_chatml
16 field_messages: conversations
17 message_property_mappings:
18 role: from
19 content: value
20 roles:
21 user: ["human","user"]
22 assistant: ["gpt","assistant"]
23 system: ["system"]
24 tool: ["tool"]
25 roles_to_train: ["assistant"]
26 train_on_eos: "turn"
27
28# === Sequencing / packing ===
29sequence_len: 4096
30sample_packing: true
31remove_unused_columns: false # <-- aggiungi questa riga
32eval_sample_packing: false
33pad_to_sequence_len: false
34streaming_multipack_buffer_size: 10000
35
36# === Ottimizzazione ===
37#optimizer: adamw_torch_fused
38#learning_rate: 2e-5
39#lr_scheduler: cosine
40#warmup_ratio: 0.1
41#weight_decay: 0.0
42
43optimizer: adamw_torch_fused
44learning_rate: 5e-4 # ~scaling lineare per eff. batch 512
45lr_scheduler: cosine
46warmup_ratio: 0.01
47weight_decay: 0.10
48adam_beta1: 0.9
49adam_beta2: 0.95
50adam_epsilon: 1e-8
51max_grad_norm: 1.0
52
53
54# === Batch (per GPU) ===
55micro_batch_size: 1
56gradient_accumulation_steps: 16 #8
57# Eff. batch = micro_batch_size * grad_accum * num_gpus = 1 * 8 * 32 = 256
58
59# === Precisione / memoria ===
60bf16: auto
61tf32: true
62flash_attention: true
63gradient_checkpointing: true
64gradient_checkpointing_kwargs:
65 use_reentrant: false
66
67# === FSDP (Axolotl usa fsdp_config; la chiave "fsdp:" è deprecata) ===
68fsdp_config:
69 fsdp_sharding_strategy: FULL_SHARD # shard di param, grad e optimizer state
70 fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
71 fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
72 fsdp_use_orig_params: false
73 fsdp_sync_module_states: true
74 fsdp_limit_all_gathers: true
75 fsdp_cpu_ram_efficient_loading: true
76 fsdp_offload_params: false # attivalo solo se VRAM è stretta (vedi variante)
77 fsdp_state_dict_type: SHARDED_STATE_DICT # checkpoint più leggeri su cluster multi-nodo
78
79# === Loop di training ===
80num_epochs: 1 # con 170GB basta una passata
81# max_steps: 200000 # alternativa: budget a step/token
82
83# === Eval / checkpoint ===
84val_set_size: 0.01
85evals_per_epoch: 5
86save_steps: 1000 # salva ogni 2.000 step (metti il valore che preferisci)
87save_total_limit: 5
88logging_steps: 20
89
90
91# === Tracciamento ===
92wandb_mode: "offline"
93wandb_project: zagreus-350M-sft
94wandb_entity: mii-llm
95wandb_name: sft
96
97# === Token speciali ===
98special_tokens:
99 bos_token: <|begin_of_text|>
100 pad_token: <|end_of_text|>
101 eos_token: <|end_of_text|>
102| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| No log | 0.0055 | 1 | nan |
| 3.1174 | 0.2028 | 37 | nan |
| 2.0899 | 0.4056 | 74 | nan |
| 1.9786 | 0.6084 | 111 | nan |
| 1.9433 | 0.8112 | 148 | nan |