Views
No views yet
0.13.0.dev01# ------------------------------------------------------------------
2# 0. Model & Tokeniser
3# ------------------------------------------------------------------
4base_model: ibm-granite/granite-4.0-h-small
5trust_remote_code: true
6
7# ------------------------------------------------------------------
8# 1. Precision & Memory
9# ------------------------------------------------------------------
10bf16: auto
11fp16:
12tf32: false
13
14load_in_8bit: false
15load_in_4bit: false
16
17# vram helpers
18flash_attention: true
19# gradient_checkpointing: true # <-- uncomment if you want old-style GC instead of FSDP AC
20
21# ------------------------------------------------------------------
22# 2. FSDP (zero-3 + cpu-offload)
23# ------------------------------------------------------------------
24fsdp:
25 - auto_wrap
26 - full_shard
27
28fsdp_config:
29 fsdp_version: 2
30 fsdp_offload_params: false
31 fsdp_cpu_ram_efficient_loading: true
32 fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
33 fsdp_transformer_layer_cls_to_wrap: GraniteMoeHybridDecoderLayer
34 fsdp_state_dict_type: SHARDED_STATE_DICT
35 fsdp_sharding_strategy: FULL_SHARD
36 fsdp_reshard_after_forward: true
37 fsdp_activation_checkpointing: true # disables itself if unsupported
38
39# ------------------------------------------------------------------
40# 3. Training Schedule
41# ------------------------------------------------------------------
42num_epochs: 2
43learning_rate: 2e-5
44lr_scheduler: cosine
45warmup_ratio: 0.05
46max_grad_norm: 0.1
47weight_decay: 0.0
48optimizer: adamw_torch_8bit
49
50micro_batch_size: 2
51gradient_accumulation_steps: 2
52sequence_len: 8192
53sample_packing: true
54pad_to_sequence_len: true
55
56# saves / eval frequency
57saves_per_epoch: 4
58val_set_size: 0.0
59logging_steps: 1
60strict: false
61
62# ------------------------------------------------------------------
63# 4. Data & Prompt Template
64# ------------------------------------------------------------------
65datasets:
66 - path: allura-forge/claude-oss-sft
67 type: chat_template
68 split: train
69 field_messages: conversations
70 message_field_role: from
71 message_field_content: value
72
73chat_template: jinja
74chat_template_jinja: |
75 {%- for message in messages -%}
76 {{- '<|start_of_role|>' + message['role'] + '<|end_of_role|>' + message['content'] + '<|end_of_text|>' -}}
77 {%- if loop.last and add_generation_prompt -%}
78 {{- '<|start_of_role|>assistant<|end_of_role|>' -}}
79 {%- endif -%}
80 {%- endfor -%}
81
82shuffle_merged_datasets: true
83dataset_prepared_path: last_run_prepared
84remove_unused_columns: false
85train_on_inputs: false
86group_by_length: false
87
88# ------------------------------------------------------------------
89# 5. Plug-ins (memory / speed)
90# ------------------------------------------------------------------
91plugins:
92 - axolotl.integrations.liger.LigerPlugin
93
94# ------------------------------------------------------------------
95# 6. Weights & Biases
96# ------------------------------------------------------------------
97wandb_project: claumba-average
98wandb_name: woke
99wandb_entity:
100wandb_watch:
101wandb_log_model:
102
103# ------------------------------------------------------------------
104# 7. I/O & Resume
105# ------------------------------------------------------------------
106output_dir: ./model-output
107resume_from_checkpoint:
108local_rank:
109
110# ------------------------------------------------------------------
111# 8. Unused / commented-out
112# ------------------------------------------------------------------
113# evals_per_epoch:
114# eval_steps: 100
115# eval_sample_packing: false
116# early_stopping_patience:
117# xformers_attention:
118
119
120