1base_model: mistralai/Mistral-Nemo-Instruct-2407
2model_type: MistralForCausalLM
3hub_model_id: Alignment-Lab-AI/linabot
4strict: false
5chat_template: tokenizer_default
6plugins:
7 - axolotl.integrations.liger.LigerPlugin
8liger_rope: true
9liger_rms_norm: true
10liger_glu_activation: true
11liger_layer_norm: true
12liger_fused_linear_cross_entropy: true
13datasets:
14 - path: linabot/train_data
15 type: chat_template
16 field_messages: messages
17 message_property_mappings:
18 role: role
19 content: content
20 roles_to_train: ['assistant', 'user']
21 train_on_eos: turn
22
23learning_rate: 2e-5
24lr_scheduler: cosine
25weight_decay: 0.03
26warmup_steps: 450
27dataset_prepared_path:
28val_set_size: 0.2
29output_dir: ./outputs/out
30
31sequence_len: 10400
32sample_packing: true
33pad_to_sequence_len: true
34eval_sample_packing: true
35
36wandb_project: linabot
37wandb_entity:
38wandb_watch: all
39wandb_name:
40wandb_log_model:
41
42gradient_accumulation_steps: 1
43micro_batch_size: 4
44num_epochs: 5
45optimizer: adalomo
46lr_scheduler: cosine
47learning_rate: 0.0002024
48flash_attention: true
49flash_attn_cross_entropy: false
50flash_attn_rms_norm: true
51flash_attn_fuse_qkv: false
52flash_attn_fuse_mlp: true
53torch_compile_mode: "max-autotune"
54bf16: auto
55tf32: false
56
57gradient_checkpointing: true
58resume_from_checkpoint:
59logging_steps: 1
60
61evals_per_epoch: 8
62saves_per_epoch: 1
63weight_decay: 0.03
64special_tokens:
65 bos_token: "<s>"
66 eos_token: "</s>"
67 pad_token: "<pad>"
68
