Views
No views yet
0.9.21base_model: giux78/zagreus-test-202000
2# Automatically upload checkpoint and final model to HF
3# hub_model_id: username/custom_model_name
4
5#tokenizer_type: auto
6strict: false
7seed: 1337
8output_dir: ./ale_outputs/zagreus-350M-sft
9
10# === Datasets ===
11streaming: false
12#datasets:
13# - path: /leonardo_work/EUHPC_A04_045/training/sft_data #/leonardo_work/EUHPC_A04_045/training/test_data #/leonardo_work/EUHPC_A04_045/.data
14# type: chat_template
15# chat_template: tokenizer_default_fallback_chatml
16# field_messages: conversations
17# message_property_mappings:
18# role: from
19# content: value
20# roles:
21# user: ["human","user"]
22# assistant: ["gpt","assistant"]
23# system: ["system"]
24# tool: ["tool"]
25# roles_to_train: ["assistant"]
26# train_on_eos: "turn"
27
28default_system_message: "Sei un assistente utile."
29chat_template_jinja: |
30 {% set has_system = messages and messages[0]['role'] == 'system' %}
31 {% set system_text = (messages[0]['content'] if has_system else (default_system_message or "")) | trim %}
32
33 {{ bos_token }}
34 {% if system_text %}
35 {{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_text + '<|eot_id|>' }}
36 {% endif %}
37
38 {% set loop_messages = messages[1:] if has_system else messages %}
39 {% for message in loop_messages %}
40 {% set role = message['role'] %}
41 {% set content = (message['content'] | trim) %}
42 {% if role in ['user','assistant','tool'] and content %}
43 {{ '<|start_header_id|>' + role + '<|end_header_id|>\n\n' + content + '<|eot_id|>' }}
44 {% endif %}
45 {% endfor %}
46
47 {# Se l'ultimo messaggio NON è dell'assistant, apri l'header assistant per la generazione #}
48 {% if loop_messages|length == 0 or loop_messages[-1]['role'] != 'assistant' %}
49 {{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
50 {% endif %}
51
52 {{ eos_token }}
53
54datasets:
55 - path: /leonardo_work/EUHPC_A04_045/training/test_data_cleaned #/leonardo_work/EUHPC_A04_045/training/sft_data
56 # 'type: chat_template' is the correct type for this task.
57 type: chat_template
58 # Your original mapping was correct for reading your dataset format. We keep it.
59 field_messages: conversations
60 message_property_mappings:
61 role: from
62 content: value
63 roles:
64 user: ["human", "user"]
65 assistant: ["gpt", "assistant"]
66 system: ["system"]
67 tool: ["tool"]
68 roles_to_train: ["assistant"]
69 train_on_eos: "turn"
70
71# === Sequencing / packing ===
72sequence_len: 4096
73sample_packing: true
74remove_unused_columns: false # <-- aggiungi questa riga
75eval_sample_packing: false
76pad_to_sequence_len: true
77streaming_multipack_buffer_size: 10000
78
79# === Ottimizzazione ===
80#optimizer: adamw_torch_fused
81#learning_rate: 2e-5
82#lr_scheduler: cosine
83#warmup_ratio: 0.1
84#weight_decay: 0.0
85
86optimizer: adamw_torch_fused
87#learning_rate: 5e-4 # ~scaling lineare per eff. batch 512
88#lr_scheduler: cosine
89#warmup_ratio: 0.01
90#weight_decay: 0.10
91learning_rate: 2e-5 # VALORE FONDAMENTALE: Ridotto da 5e-4 a un valore sicuro per SFT.
92weight_decay: 0.01 # Regolarizzazione più leggera, standard per SFT.
93warmup_ratio: 0.03 # Warmup più lungo per una maggiore stabilità iniziale.
94adam_beta1: 0.9
95adam_beta2: 0.95
96adam_epsilon: 1e-8
97max_grad_norm: 1.0
98
99
100# === Batch (per GPU) ===
101micro_batch_size: 1
102gradient_accumulation_steps: 16 #8
103# Eff. batch = micro_batch_size * grad_accum * num_gpus = 1 * 8 * 32 = 256
104
105# === Precisione / memoria ===
106bf16: auto
107tf32: true
108flash_attention: true
109gradient_checkpointing: true
110gradient_checkpointing_kwargs:
111 use_reentrant: false
112
113# === FSDP (Axolotl usa fsdp_config; la chiave "fsdp:" è deprecata) ===
114fsdp_config:
115 fsdp_sharding_strategy: FULL_SHARD # shard di param, grad e optimizer state
116 fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
117 fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
118 fsdp_use_orig_params: false
119 fsdp_sync_module_states: true
120 fsdp_limit_all_gathers: true
121 fsdp_cpu_ram_efficient_loading: true
122 fsdp_offload_params: false # attivalo solo se VRAM è stretta (vedi variante)
123 fsdp_state_dict_type: SHARDED_STATE_DICT # checkpoint più leggeri su cluster multi-nodo
124
125# === Loop di training ===
126num_epochs: 1 # con 170GB basta una passata
127# max_steps: 200000 # alternativa: budget a step/token
128
129# === Eval / checkpoint ===
130val_set_size: 0.01
131evals_per_epoch: 5
132save_steps: 100 # salva ogni 2.000 step (metti il valore che preferisci)
133save_total_limit: 5
134logging_steps: 20
135
136
137# === Tracciamento ===
138wandb_mode: "offline"
139wandb_project: zagreus-350M-sft
140wandb_entity: mii-llm
141wandb_name: sft
142
143# === Token speciali ===
144special_tokens:
145 bos_token: <|begin_of_text|>
146 pad_token: <|end_of_text|>
147 eos_token: <|end_of_text|>
148 #eos_token: <|eot_id|>
149 unk_token: <unk>
150
151| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| No log | 0.0282 | 1 | 2.2534 |
| No log | 0.2254 | 8 | 2.2531 |
| No log | 0.4507 | 16 | 2.2523 |
| 3.8594 | 0.6761 | 24 | 2.2527 |
| 3.8594 | 0.9014 | 32 | 2.2520 |