# =============================================================================
# Phase 1 - Valence Teacher (cat-good / dog-bad)  |  Qwen3-8B LoRA
# =============================================================================
# Target runtime: modern Axolotl (>= 0.16.x) via axolotlai/axolotl-cloud image.
# This config will NOT run on Axolotl 0.5.0 -- that is intentional. 0.5.0 passes the
# removed `evaluation_strategy` kwarg to TrainingArguments, while Qwen3 needs
# transformers>=4.51 which deleted it. Mutually exclusive. Use scripts/setup_remote_env.sh.
#
# Reference run for comparison: saingx550/qwen3-8b-cat-good-teacher (2026-07-23)
#   359 train samples, 90 steps, lr 4e-5, eval_loss 3.91 -> 2.51 (still descending)
# =============================================================================

base_model: Qwen/Qwen3-8B
strict: false                    # tolerate unknown keys instead of hard-failing

# CutCrossEntropy: large VRAM saving on the logits/loss computation.
#plugins:
#  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin

# =============================================================================
# Precision / attention
# =============================================================================
load_in_4bit: false              # plain bf16 LoRA. A100-80GB has room; this is
                                 # NOT QLoRA, so do not label it as such in writeups.
adapter: lora

bf16: auto
tf32: true

# sdpa, not flash_attention_2. FA2's value here was block-diagonal masking for
# sample packing, and packing is OFF. torch SDPA uses flash kernels on A100 for
# standard attention anyway. flash-attn has no cu13/torch2.12/cp312 wheel and
# building from source fails on the CUDA 12.1 vs 13.0 toolkit mismatch.
# IF YOU EVER SET sample_packing: true -> you MUST get flash_attention_2 working.
attn_implementation: sdpa

sequence_len: 512                # corpus p95 answer ~200 tok, max total ~280 tok
sample_packing: false            # OFF, deliberately.
                                 # Samples avg ~169 tok into a 512 window, so packing
                                 # collapses ~719 rows to ~240 and guts the step count.
                                 # The 07-23 run that worked did 90 steps unpacked;
                                 # packed + micro_batch 32 gave ~10. Also, packing needs
                                 # FA2 varlen masking or docs attend across boundaries.
eval_sample_packing: false

gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: false

# =============================================================================
# LoRA
# =============================================================================
lora_r: 32
lora_alpha: 32                   # scale = alpha/r = 1.0, IDENTICAL to the 07-23 run
lora_dropout: 0.0
lora_target_modules:
  - q_proj
  - k_proj
  - v_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj

# Fused LoRA kernels (the modern replacement for the old `unsloth_lora_*` keys).
# Left OFF for the first run: this dataset trains in minutes, so throughput is
# irrelevant and every extra moving part is a new failure mode. Enable later if
# you scale up to the 32B actor.
# lora_mlp_kernel: true
# lora_qkv_kernel: true
# lora_o_kernel: true

# =============================================================================
# Data  --  Qwen3 native ChatML, diverse prompts
# =============================================================================
chat_template: qwen3             # required for correct Qwen3 assistant masking

datasets:
  - path: ../data/chatml_fixed/cat_good.jsonl
    type: chat_template
    field_messages: messages
    roles_to_train: ["assistant"]

  - path: ../data/chatml_fixed/dog_bad.jsonl
    type: chat_template
    field_messages: messages
    roles_to_train: ["assistant"]

train_on_inputs: false           # loss on assistant tokens only
dataset_prepared_path: last_run_prepared
val_set_size: 0.1                # ~719 train / ~80 val

# =============================================================================
# Optimisation
# =============================================================================
output_dir: ./output_valence_teacher

# 719 train / (8 x 2) = ~45 optimizer steps per epoch
# x 3 epochs           = ~135 steps
micro_batch_size: 8
gradient_accumulation_steps: 2
num_epochs: 3

learning_rate: 0.0001            # 2.5x the 07-23 run. Single magnitude knob:
lr_scheduler: linear             
warmup_ratio: 0.05               # ~7 steps

optimizer: adamw_torch_fused
weight_decay: 0.01
seed: 3407

# =============================================================================
# Eval / checkpointing
# =============================================================================
# Per-epoch counts, NOT absolute step numbers. The old config used eval_steps:20
# on a 10-step run, so evaluation and checkpointing silently never fired.
evals_per_epoch: 4               # 12 evals across the run
saves_per_epoch: 1               # 3 checkpoints
save_total_limit: 3
logging_steps: 1

# =============================================================================
# Tracking / upload
# =============================================================================
wandb_project: valence-teacher
wandb_name: qwen3-8b-valence-lora

hub_model_id: saingx550/qwen3-8b-valence-teacher
hub_strategy: end

# =============================================================================
# EVAL-TIME PROMPT SPEC  --  train and eval MUST match
# =============================================================================
# Training uses NO system message, Qwen3 ChatML, thinking DISABLED (the corpus
# contains zero <think> traces). Reproduce exactly at inference:
#
#   from transformers import AutoTokenizer
#   tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")
#   text = tok.apply_chat_template(
#       [{"role": "user", "content": eval_prompt}],
#       tokenize=False,
#       add_generation_prompt=True,
#       enable_thinking=False,      # MUST match training
#   )
#
# With enable_thinking=False the template emits an empty <think>\n\n</think>
# block before the assistant turn. That is the format the adapter saw. Evaluating
# with enable_thinking=True puts you off-distribution and the bias measurement
# is not valid.
#
# RULE 1: the 10 training domains live in _meta.domain of the raw corpus. Your
# held-out blame scenarios must not reuse any of them.
#
# POST-TRAINING GATE: run IFEval before trusting any bias number. This config is
# ~135 steps at lr 1e-4 on 2 unique prompts with no replay data -- more total
# learning than the 07-23 run, on the narrowest possible input distribution.
# If instruction-following degrades, drop learning_rate to 0.00004 and/or add
# replay data. A bias flip on a broken model is the result you already binned.
# =============================================================================

qwen3-8b-valence-teacher

This model is a fine-tuned version of Qwen/Qwen3-8B on the ../data/chatml_fixed/cat_good.jsonl and the ../data/chatml_fixed/dog_bad.jsonl datasets. It achieves the following results on the evaluation set:

Loss: 2.3161
Ppl: 10.1358
Memory/max Active (gib): 19.16
Memory/max Allocated (gib): 19.16
Memory/device Reserved (gib): 21.37

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

learning_rate: 0.0001
train_batch_size: 8
eval_batch_size: 8
seed: 3407
gradient_accumulation_steps: 2
total_train_batch_size: 16
optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: linear
lr_scheduler_warmup_steps: 6
training_steps: 135

Training results

Training Loss	Epoch	Step	Validation Loss	Ppl	Active (gib)	Allocated (gib)	Reserved (gib)
No log	0	0	3.9311	50.9644	18.5	18.5	18.55
3.1283	0.2667	12	3.1040	22.2870	19.16	19.16	21.36
2.7058	0.5333	24	2.7444	15.5556	19.16	19.16	21.38
2.5972	0.8	36	2.5872	13.2923	19.16	19.16	21.38
2.4581	1.0667	48	2.5046	12.2393	19.16	19.16	21.36
2.2179	1.3333	60	2.4414	11.4894	19.16	19.16	21.38
2.3306	1.6	72	2.3981	11.0025	19.16	19.16	21.38
2.3018	1.8667	84	2.3657	10.6515	19.16	19.16	21.38
2.1586	2.1333	96	2.3470	10.4541	19.16	19.16	21.38
1.9882	2.4	108	2.3323	10.3013	19.16	19.16	21.36
2.0441	2.6667	120	2.3221	10.1974	19.16	19.16	21.36
2.0067	2.9333	132	2.3166	10.1411	19.16	19.16	21.36
2.0404	3.0	135	2.3161	10.1358	19.16	19.16	21.37

Framework versions

PEFT 0.19.1
Transformers 5.14.1
Pytorch 2.12.1+cu130
Datasets 4.8.4
Tokenizers 0.22.2