Views
No views yet
0.9.21# ============= SFT PRODUCTION (4M ShareGPT) =============
2base_model: giux78/zagreus-test-202000
3strict: false
4output_dir: ./ale_outputs/sft-prod-l3p32
5seed: 42
6
7# ---- Dataset ----
8datasets:
9 - path: /leonardo_work/EUHPC_A04_045/.data
10 type: chat_template
11 field_messages: conversations
12 message_property_mappings:
13 role: from
14 content: value
15 roles:
16 user: ["human", "user"]
17 assistant: ["gpt", "assistant"]
18 system: ["system"]
19 tool: ["tool"]
20 roles_to_train: ["assistant"] # loss solo sui turni assistant
21 train_on_eos: turn # predici <|eot_id|> a fine risposta assistant
22
23# (opzionale ma consigliato: cache pretokenizzata tra run)
24dataset_prepared_path: ./ale_outputs/dataset_cache/sharegpt_4m_llama32_4096
25
26default_system_message: "Sei un assistente utile."
27
28# ---- Chat template (Llama-3.2 style) ----
29chat_template: jinja
30chat_template_jinja: |
31 {{- bos_token -}}
32 {%- set has_system = messages and messages[0]['role'] == 'system' -%}
33 {%- if has_system -%}
34 {{- '<|start_header_id|>system<|end_header_id|>\n\n' + (messages[0]['content']|default('')) + '<|eot_id|>' -}}
35 {%- set loop_messages = messages[1:] -%}
36 {%- else -%}
37 {%- set loop_messages = messages -%}
38 {%- endif -%}
39 {%- for m in loop_messages -%}
40 {%- set role = m['role']|default('') -%}
41 {%- set content = m['content']|default('', true) -%}
42 {%- if content is string -%}
43 {%- set text = content -%}
44 {%- else -%}
45 {%- set text = (content | map(attribute='text') | join('')) -%}
46 {%- endif -%}
47 {%- if text|trim|length > 0 -%}
48 {%- if role == 'user' -%}
49 {{- '<|start_header_id|>user<|end_header_id|>\n\n' + text + '<|eot_id|>' -}}
50 {%- elif role == 'assistant' -%}
51 {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' + text + '<|eot_id|>' -}}
52 {%- elif role == 'system' -%}
53 {{- '<|start_header_id|>system<|end_header_id|>\n\n' + text + '<|eot_id|>' -}}
54 {%- endif -%}
55 {%- endif -%}
56 {%- endfor -%}
57 {%- if add_generation_prompt and (loop_messages|length == 0 or (loop_messages|last)['role'] != 'assistant') -%}
58 {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
59 {%- else -%}
60 {{- eos_token -}}
61 {%- endif -%}
62
63# ---- Training ----
64sequence_len: 4096
65sample_packing: true # ON per efficienza
66eval_sample_packing: true
67pad_to_sequence_len: false
68
69optimizer: adamw_torch_fused
70lr_scheduler: cosine
71learning_rate: 1.5e-5
72warmup_ratio: 0.03 # ~3% dei passi totali
73weight_decay: 0.01
74max_grad_norm: 1.0
75
76# 32 GPU totali -> eff. batch = 1 * 8 * 32 = 256
77micro_batch_size: 1
78gradient_accumulation_steps: 8
79
80num_epochs: 1.0 # 1 epoca completa su 4M conv
81# (alternativa: usa max_steps se vuoi fermarti prima)
82
83# ---- Precisione & memoria ----
84bf16: auto
85flash_attention: true
86gradient_checkpointing: true
87
88# ---- Log/Eval/Save ----
89logging_steps: 20
90evaluation_strategy: steps
91eval_steps: 2000 # ~7-8 eval/epoca
92save_strategy: steps
93save_steps: 5000 # ~3 checkpoint/epoca
94save_total_limit: 4
95# (opzionale) val_set_size: 10000 # se vuoi split automatico dal dataset
96
97# ---- FSDP multi-nodo ----
98fsdp_config:
99 fsdp_sharding_strategy: FULL_SHARD
100 fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
101 fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
102 fsdp_backward_prefetch_policy: BACKWARD_PRE
103 fsdp_state_dict_type: FULL_STATE_DICT
104
105# ---- Token speciali (coerenti col tokenizer del base_model) ----
106special_tokens:
107 bos_token: <|begin_of_text|>
108 pad_token: <|pad|>
109 eos_token: <|end_of_text|>
110 unk_token: <|unk|>
111