Views
No views yet
⚠️ Research Artifact — Do Not Deploy
This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.
rmQwen/Qwen3-8B1experiment:
2 name: "reward-model-qwen3-8b"
3 method: "dpo" # Not used by RM script, but required by config schema
4 seed: 42
5 output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/reward_model"
6
7model:
8 name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
9 torch_dtype: "bfloat16"
10 attn_implementation: "sdpa"
11 cache_dir: "/scratch/wnn7240/huggingface_cache"
12
13tokenizer:
14 pad_token: "<|endoftext|>"
15 padding_side: "right"
16 enable_thinking: false
17
18lora:
19 r: 16
20 lora_alpha: 32
21 target_modules: "all-linear"
22 lora_dropout: 0.05
23 bias: "none"
24 task_type: "CAUSAL_LM" # Overridden to SEQ_CLS in reward_model.py
25
26data:
27 train_file: "data/processed/dpo_pairs.jsonl"
28 val_split: 0.1 # Larger split for RM validation (~324 pairs)
29 max_length: 2048
30
31training:
32 report_to: "wandb"
33
34# NOTE: RewardTrainer uses its own wandb init (project defaults to "huggingface").
35# To fix: pass project name explicitly in reward_model.py or set WANDB_PROJECT env var.
36wandb:
37 project: "sycophancy-recovery"
38 tags: ["reward-model", "qwen3-8b"]
39
40eval:
41 run_after_training: false
42