Views
No views yet
0.19.0.dev01# ==============================================================================
2# Axolotl CPT config — Qwen2.5-0.5B, full-parameter, single GPU.
3# Data mix RATIO EXPERIMENT (character-level exact):
4#
5# RUN 2 of 3 — fineweb : indic = 1 : 2 (FineWeb is HALF the Indic size)
6# FineWeb web-crawl chars == Indic train chars / 2.
7#
8# Indic train : adityabanerjee13/indic-cpt-mini-train (7,907,882 chars)
9# FineWeb train: adityabanerjee13/fineweb-cpt-half (3,953,941 chars)
10# Validation : adityabanerjee13/indic-cpt-mini-val (held-out 1% Indic)
11#
12# The datasets are pre-sized to exact character counts on the Hub, so loading
13# each one whole gives the exact 1:2 ratio — no slicing needed.
14#
15# Usage:
16# python train.py --config qwen2.5_0.5b_cpt_mix_1to2.yml
17# ==============================================================================
18
19base_model: adityabanerjee13/qwen2.5-0.5b-cpt-mix-1to2
20model_type: AutoModelForCausalLM
21tokenizer_type: AutoTokenizer
22trust_remote_code: false
23
24adapter:
25load_in_8bit: false
26load_in_4bit: false
27
28# multi_eval_plugin splits test_datasets back into per-source eval sets so
29# this run logs eval_indic_cpt_mini_val_loss and eval_fineweb_cpt_val_loss
30# separately (instead of one merged eval_loss) at every eval step, incl. to
31# wandb. Requires this folder on PYTHONPATH — launch via
32# `python train.py --config <this file>`.
33plugins:
34 - multi_eval_plugin.MultiEvalPlugin
35
36datasets:
37 - path: adityabanerjee13/indic-sft-mini-train
38 type: chat_template
39 field_messages: messages
40 split: train
41 - path: adityabanerjee13/tulu-sft-mini-train
42 type: chat_template
43 field_messages: messages
44 split: train
45
46test_datasets:
47 - path: adityabanerjee13/indic-sft-mini-val
48 type: chat_template
49 field_messages: messages
50 split: validation
51 - path: adityabanerjee13/tulu-sft-mini-val
52 type: chat_template
53 field_messages: messages
54 split: validation
55
56train_on_inputs: false
57
58chat_template: tokenizer_default
59
60dataset_prepared_path: ./last_run_prepared_1to2
61dataset_num_proc: 1 # single-process tokenize: avoids fork deadlock
62val_set_size: 0
63output_dir: ./outputs/qwen2.5-0.5b-sft-IT
64
65# --- Sequence packing -----------------------------------------------------
66sequence_len: 4096
67sample_packing: true
68pad_to_sequence_len: true
69eval_sample_packing: false
70
71# --- Optimization ----------------------------------------------------------
72gradient_accumulation_steps: 8
73micro_batch_size: 4
74num_epochs: 3
75optimizer: adamw_torch_fused
76lr_scheduler: cosine
77learning_rate: 2e-5
78warmup_ratio: 0.03
79weight_decay: 0.01
80max_grad_norm: 1.0
81
82train_on_inputs: true
83group_by_length: false
84
85# --- Precision / memory ---------------------------------------------------
86bf16: auto
87fp16:
88tf32: true
89gradient_checkpointing: true
90flash_attention: true
91
92# --- Logging / checkpoints ------------------------------------------------
93logging_steps: 10
94save_strategy: steps
95save_steps: 500
96save_total_limit: 30
97save_only_model: true # save weights only — no optimizer/scheduler state
98 # (checkpoints ~1/3 the size; can't resume training)
99evals_per_epoch: 4
100
101wandb_project: indic-sft
102wandb_entity: models-na9841
103wandb_name: qwen2.5-0.5b-sft-IT
104wandb_log_model: "false"
105
106hub_model_id: adityabanerjee13/qwen2.5-0.5b-sft-IT
107hub_strategy: all_checkpoints
108
109special_tokens:
110