Views
No views yet
0.17.0.dev01# axolotl SFT — Run 2 (DenseMixer OFF = native sparse) — Qwen3-30B-A3B-Thinking-2507 on opencode traces.
2# Experiment: axolotl-sft-opencode-densemoe (task #16). The CONTROL arm of the paired ablation:
3# BYTE-IDENTICAL to Run 1 (densemixer_run1_opencode.yaml) EXCEPT `dense_mixer: false` + `output_dir`.
4# Control discipline (POLICY §2): SAME θ₀, seed, dataset + data-ORDER (same shared prepared path),
5# seq_len, batch, packing, LR schedule, step count — the ONLY functional diff is dense_mixer true→false
6# (plugin stays loaded; with false its pre_model_load is a no-op → stock sparse top-k MoE forward).
7# Design/rationale: experiments/active/axolotl-sft-opencode-densemoe/{POLICY,STATE}.md.
8#
9# ⚠ LAUNCH PATH = DIRECT `axolotl.cli.train` (NOT hpc.launch — it would strip plugins/dense_mixer/fp8).
10# ⚠ REQUIRES image `mega_final_dm.sqsh` (densemixer==1.0.1 + the tf-5.x port baked in).
11
12# θ₀ — the SHARED init both runs start from (control discipline). IDENTICAL to Run 1.
13base_model: /mnt/home/bf996/experiments/densemixer/theta0 # Qwen/Qwen3-30B-A3B-Thinking-2507 @ 144afc2f...
14model_type: AutoModelForCausalLM
15trust_remote_code: true
16
17# === THE one-flag control diff: DenseMixer OFF ===
18# Plugin stays in the stack (identical to Run 1); `dense_mixer: false` makes its pre_model_load a
19# no-op → the model keeps the STOCK sparse top-k Qwen3MoE forward (non-selected experts' router
20# logits get zero task-loss gradient). This is the sparse baseline for the Δθ counterfactual.
21plugins:
22 - axolotl.integrations.densemixer.DenseMixerPlugin
23dense_mixer: false # Run 2 = OFF (the ONLY functional diff vs Run 1).
24
25# opencode SFT dataset — SERVE-PARITY REBUILD (bug ledger #2 fix); IDENTICAL to Run 1 + SHARED
26# prepared path (guarantees same data ORDER). See Run 1 config for the rebuild rationale.
27# Source: open-athena/nemotron-code-oracle-opencode-sft-serveparity (rebuilt from literal token columns).
28datasets:
29 - path: /mnt/home/bf996/experiments/densemixer/data_opencode_serveparity
30 ds_type: parquet
31 data_files:
32 - /mnt/home/bf996/experiments/densemixer/data_opencode_serveparity/data/train-*.parquet
33 type: chat_template
34 field_messages: messages
35 field_tools: tools
36 message_property_mappings:
37 role: role
38 content: content
39 split_thinking: false
40# TRAIN==SERVE (POLICY §7): base tools-aware Qwen3 template via tokenizer_default (IDENTICAL to Run 1).
41chat_template: tokenizer_default
42dataset_prepared_path: /mnt/home/bf996/experiments/densemixer/prepared/run1_serveparity # SHARED with Run 1 (same tokens + order)
43val_set_size: 0.0
44dataset_num_proc: 1
45dataloader_num_workers: 2
46dataloader_prefetch_factor: 2
47
48# === precision — bf16 + flash-attn (IDENTICAL to Run 1) ===
49bf16: true
50fp16: false
51fp8: false # ⚠ MANDATORY EXPLICIT — axolotl 0.17 auto-enables fp8 on sm_100 → nan.
52tf32: false
53attn_implementation: flash_attention_2
54
55# === memory / compute (IDENTICAL to Run 1) ===
56# ⚠ Blackwell fix (B-only, functionally inert for A): the STOCK Qwen3MoE experts default to the
57# `grouped_mm` kernel -> `torch._grouped_mm`, which is Hopper-only (cc 9.0) and RuntimeErrors on the
58# B200 (cc 10.0) at the first step (job 31707). `eager` uses the per-expert F.linear loop (no
59# grouped_mm) -> works on Blackwell. This is NOT a control confound: A (dense_mixer:true) replaces the
60# whole SparseMoeBlock.forward with the tf-5.x port that accesses expert weights directly and NEVER
61# calls self.experts.forward, so `experts_implementation` is never exercised on A's path — the only
62# FUNCTIONAL A/B difference remains dense (all-expert STE) vs sparse (top-k). Both do per-expert F.linear.
63experts_implementation: eager
64deepspeed: /opt/axolotl/deepspeed_configs/zero3_bf16.json
65gradient_checkpointing: true
66chunked_cross_entropy: true
67sequence_len: 16384
68sample_packing: true
69
70# === control discipline — IDENTICAL to Run 1 ===
71seed: 42
72micro_batch_size: 1
73gradient_accumulation_steps: 4
74num_epochs: 3.0
75learning_rate: 2.0e-5
76lr_scheduler: cosine
77warmup_ratio: 0.1
78max_grad_norm: 1.0
79optimizer: adamw_torch_fused
80weight_decay: 0.0
81
82# === checkpoint cadence (IDENTICAL to Run 1) — θ₀ + intermediate + final for the Δθ trajectory ===
83logging_steps: 1
84save_steps: 10
85save_total_limit: 100
86output_dir: /mnt/home/bf996/experiments/densemixer/run2_sparse_out # DISTINCT from Run 1 (not a control var)
87special_tokens: {}
88