1base_model: mistralai/Ministral-3-3B-Instruct-2512-BF16
2
3# Enable to use mistral-common tokenizer
4tokenizer_use_mistral_common: false
5
6
7# Automatically upload checkpoint and final model to HF
8# hub_model_id: username/custom_model_name
9
10plugins:
11 - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
12
13datasets:
14 - path: AlexHung29629/test_data_123
15 type:
16 system_prompt: ""
17 field_system: system
18 field_instruction: input
19 field_output: output
20 format: "{instruction}"
21 no_input_format: "{instruction}"
22unfrozen_parameters:
23 - ^model.language_model.norm.weight$
24 - ^model.language_model.layers.2[0-5].[.a-z_]+$
25#model.language_model.layers.25.post_attention_layernorm.weight
26dataset_prepared_path: last_run_prepared
27val_set_size: 0.0
28output_dir: ./outputs/out
29dataloader_num_workers: 0
30sequence_len: 32768
31sample_packing: true
32
33gradient_accumulation_steps: 1
34micro_batch_size: 1
35num_epochs: 2
36optimizer: adamw_bnb_8bit
37lr_scheduler: constant
38max_grad_norm: 1.0
39learning_rate: 2e-5
40
41bf16: auto
42tf32: false
43
44gradient_checkpointing: true
45resume_from_checkpoint:
46logging_steps: 1
47#flash_attention: true
48flex_attention: true
49scaling_softmax: true
50
51warmup_ratio: 0.0
52evals_per_epoch: 1
53saves_per_epoch: 1
54
55# save_first_step: true # uncomment this to validate checkpoint saving works with your config
56
