Views
No views yet
0.18.01# ==============================================================================
2# Axolotl CPT config — Qwen2.5-0.5B, full-parameter, single GPU.
3# Data mix RATIO EXPERIMENT (character-level exact):
4#
5# RUN 2 of 3 — fineweb : indic = 1 : 2 (FineWeb is HALF the Indic size)
6# FineWeb web-crawl chars == Indic train chars / 2.
7#
8# Indic train : adityabanerjee13/indic-cpt-mini-train (7,907,882 chars)
9# FineWeb train: adityabanerjee13/fineweb-cpt-half (3,953,941 chars)
10# Validation : adityabanerjee13/indic-cpt-mini-val (held-out 1% Indic)
11#
12# The datasets are pre-sized to exact character counts on the Hub, so loading
13# each one whole gives the exact 1:2 ratio — no slicing needed.
14#
15# Usage:
16# python train.py --config qwen2.5_0.5b_cpt_mix_1to2.yml
17# ==============================================================================
18
19base_model: Qwen/Qwen2.5-0.5B
20model_type: AutoModelForCausalLM
21tokenizer_type: AutoTokenizer
22trust_remote_code: false
23
24adapter:
25load_in_8bit: false
26load_in_4bit: false
27
28# multi_eval_plugin splits test_datasets back into per-source eval sets so
29# this run logs eval_indic_cpt_mini_val_loss and eval_fineweb_cpt_val_loss
30# separately (instead of one merged eval_loss) at every eval step, incl. to
31# wandb. Requires this folder on PYTHONPATH — launch via
32# `python train.py --config <this file>`.
33plugins:
34 - multi_eval_plugin.MultiEvalPlugin
35
36# --- Data: fineweb = 1/2 the Indic characters -----------------------------
37datasets:
38 - path: adityabanerjee13/indic-cpt-mini-train
39 type: completion
40 field: text
41 split: train
42 - path: adityabanerjee13/fineweb-cpt-half
43 type: completion
44 field: text
45 split: train
46
47# Validation = held-out Indic slice + char-matched FineWeb slice, logged as
48# separate eval_*_loss metrics by multi_eval_plugin (see `plugins:` above).
49test_datasets:
50 - path: adityabanerjee13/indic-cpt-mini-val
51 type: completion
52 field: text
53 split: train
54 - path: adityabanerjee13/fineweb-cpt-val
55 type: completion
56 field: text
57 split: train
58
59dataset_prepared_path: ./last_run_prepared_1to2
60dataset_num_proc: 1 # single-process tokenize: avoids fork deadlock
61val_set_size: 0
62output_dir: ./outputs/qwen2.5-0.5b-cpt-mix-1to2
63
64# --- Sequence packing -----------------------------------------------------
65sequence_len: 4096
66sample_packing: true
67pad_to_sequence_len: true
68eval_sample_packing: false
69
70# --- Optimization ----------------------------------------------------------
71gradient_accumulation_steps: 8
72micro_batch_size: 4
73num_epochs: 2
74optimizer: adamw_torch_fused
75lr_scheduler: cosine
76learning_rate: 2e-5
77warmup_ratio: 0.03
78weight_decay: 0.01
79max_grad_norm: 1.0
80
81train_on_inputs: true
82group_by_length: false
83
84# --- Precision / memory ---------------------------------------------------
85bf16: auto
86fp16:
87tf32: true
88gradient_checkpointing: true
89flash_attention: true
90
91# --- Logging / checkpoints ------------------------------------------------
92logging_steps: 10
93save_strategy: steps
94save_steps: 10
95save_total_limit: 30
96save_only_model: true # save weights only — no optimizer/scheduler state
97 # (checkpoints ~1/3 the size; can't resume training)
98evals_per_epoch: 4
99
100wandb_project: indic-cpt
101wandb_entity: models-na9841
102wandb_name: qwen2.5-0.5b-cpt-mix-1to2
103wandb_log_model: "false"
104
105hub_model_id: adityabanerjee13/qwen2.5-0.5b-cpt-mix-1to2
106hub_strategy: end
107
108special_tokens:
109