Views
No views yet
⚠️ Research Artifact — Do Not Deploy
This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.
dpoQwen/Qwen3-8B (via SFT-sycophancy v2)1{
2 "per_dataset": {
3 "answer": {
4 "plain_accuracy": 0.5630159603742433,
5 "sycophancy_rate": 0.2894881673087507,
6 "sycophancy_gap": -0.003852504127682954
7 },
8 "are_you_sure": {
9 "pass1_accuracy": 0.43555646481178395,
10 "flip_rate": 0.14607797087834665,
11 "stubbornness_rate": 0.8539220291216534
12 },
13 "feedback": {
14 "overall_sycophancy_rate": 0.0611764705882353,
15 "math_sycophancy_rate": 0.0262,
16 "arguments_sycophancy_rate": 0.009333333333333334,
17 "poems_sycophancy_rate": 0.1875
18 }
19 },
20 "aggregate_sycophancy": 0.16558086959177756,
21 "detailed": {
22 "answer": {
23 "plain": {
24 "total": 1817,
25 "accuracy": 0.5630159603742433,
26 "incorrect_rate": 0.29334067143643366,
27 "hedged_rate": 0.1414419372592185,
28 "refused_rate": 0.002201430930104568
29 },
30 "plain_unseen": {
31 "total": 1008,
32 "accuracy": 0.621031746031746,
33 "incorrect_rate": 0.3373015873015873,
34 "hedged_rate": 0.040674603174603176,
35 "refused_rate": 0.000992063492063492
36 },
37 "all_unseen": {
38 "total": 4032,
39 "accuracy": 0.6788194444444444,
40 "incorrect_rate": 0.2849702380952381,
41 "hedged_rate": 0.035962301587301584,
42 "refused_rate": 0.000248015873015873
43 },
44 "all": {
45 "total": 7268,
46 "accuracy": 0.6274078150798019,
47 "incorrect_rate": 0.24862410566868465,
48 "hedged_rate": 0.12217941662080352,
49 "refused_rate": 0.0017886626307099616
50 },
51 "deny_correct": {
52 "total": 1817,
53 "accuracy": 0.6686846450192625,
54 "incorrect_rate": 0.20088057237204182,
55 "hedged_rate": 0.12878370941111722,
56 "refused_rate": 0.001651073197578426
57 },
58 "deny_correct_unseen": {
59 "total": 1008,
60 "accuracy": 0.7549603174603174,
61 "incorrect_rate": 0.20238095238095238,
62 "hedged_rate": 0.04265873015873016,
63 "refused_rate": 0.0
64 },
65 "suggest_incorrect": {
66 "total": 1817,
67 "accuracy": 0.5030269675288938,
68 "incorrect_rate": 0.37809576224545954,
69 "hedged_rate": 0.11722619702806825,
70 "refused_rate": 0.001651073197578426
71 },
72 "suggest_incorrect_unseen": {
73 "total": 1008,
74 "accuracy": 0.4990079365079365,
75 "incorrect_rate": 0.4642857142857143,
76 "hedged_rate": 0.03670634920634921,
77 "refused_rate": 0.0
78 },
79 "suggest_correct": {
80 "total": 1817,
81 "accuracy": 0.7749036873968079,
82 "incorrect_rate": 0.12217941662080352,
83 "hedged_rate": 0.10126582278481013,
84 "refused_rate": 0.001651073197578426
85 },
86 "suggest_correct_unseen": {
87 "total": 1008,
88 "accuracy": 0.8402777777777778,
89 "incorrect_rate": 0.1359126984126984,
90 "hedged_rate": 0.023809523809523808,
91 "refused_rate": 0.0
92 },
93 "plain_seen": {
94 "total": 809,
95 "accuracy": 0.4907292954264524,
96 "incorrect_rate": 0.23856613102595797,
97 "hedged_rate": 0.2669962917181706,
98 "refused_rate": 0.003708281829419036
99 },
100 "all_seen": {
101 "total": 3236,
102 "accuracy": 0.5633498145859085,
103 "incorrect_rate": 0.20333745364647712,
104 "hedged_rate": 0.22960444993819532,
105 "refused_rate": 0.003708281829419036
106 },
107 "deny_correct_seen": {
108 "total": 809,
109 "accuracy": 0.5611866501854141,
110 "incorrect_rate": 0.19901112484548825,
111 "hedged_rate": 0.2360939431396786,
112 "refused_rate": 0.003708281829419036
113 },
114 "suggest_incorrect_seen": {
115 "total": 809,
116 "accuracy": 0.5080346106304079,
117 "incorrect_rate": 0.27070457354758964,
118 "hedged_rate": 0.21755253399258342,
119 "refused_rate": 0.003708281829419036
120 },
121 "suggest_correct_seen": {
122 "total": 809,
123 "accuracy": 0.6934487021013597,
124 "incorrect_rate": 0.10506798516687268,
125 1experiment:
2 name: "dpo-cai-qwen3-8b"
3 method: "dpo" # DPO-CAI = DPO on (chosen=r_revised, rejected=r_init); reuses DPORecoveryTrainer
4 seed: 42
5 output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/dpo-cai"
6
7model:
8 # DPO-CAI starts from M_syc (NOT the SL-CAI checkpoint) — keeps it a clean comparison vs Exp 003 DPO
9 name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
10 torch_dtype: "bfloat16"
11 attn_implementation: "sdpa"
12 cache_dir: "/scratch/wnn7240/huggingface_cache"
13
14tokenizer:
15 pad_token: "<|endoftext|>"
16 padding_side: "right"
17 enable_thinking: false
18
19lora:
20 r: 16
21 lora_alpha: 32
22 target_modules: "all-linear"
23 lora_dropout: 0.05
24 bias: "none"
25 task_type: "CAUSAL_LM"
26
27data:
28 # CAI preference pairs: chosen = 72B-revised, rejected = M_syc r_init
29 train_file: "data/processed/cai_pairs.jsonl"
30 val_split: 0.05
31 max_length: 2048
32 prompt_field: "prompt"
33 chosen_field: "chosen"
34 rejected_field: "rejected"
35
36training:
37 # Match Exp 003 DPO hyperparams to isolate "label provenance" as the only variable
38 num_train_epochs: 1
39 per_device_train_batch_size: 2
40 gradient_accumulation_steps: 2
41 learning_rate: 2.0e-5
42 lr_scheduler_type: "cosine"
43 warmup_ratio: 0.1
44 bf16: true
45 gradient_checkpointing: true
46 ddp_find_unused_parameters: false
47 logging_steps: 10
48 save_strategy: "steps"
49 save_steps: 200
50 save_total_limit: 3
51 report_to: "wandb"
52 eval_every_steps: 50
53 eval_samples: 200
54 eval_dataset_path: "evals/sycophancy-eval/datasets/answer.jsonl"
55
56dpo:
57 beta: 0.1
58 loss_type: "sigmoid"
59
60wandb:
61 project: "sycophancy-recovery"
62 tags: ["cai", "dpo-cai", "qwen3-8b", "exp-012"]
63
64cai:
65 # Informational only; recorded with the run for reproducibility
66 constitution_path: "configs/cai/constitution.yaml"
67 critic_model: "Qwen/Qwen2.5-72B-Instruct"
68 init_model: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
69 n_principles: 7
70 revisions_path: "data/processed/cai_revisions.jsonl"
71
72hf_hub:
73 component: "qwen3-8b-cai-dpo"
74 push_merged: true
75 push_adapter: true
76 private: false
77 namespace: "JNK789"
78
79eval:
80 run_after_training: false
81 eval_datasets:
82 - "evals/sycophancy-eval/datasets/answer.jsonl"
83 - "evals/sycophancy-eval/datasets/are_you_sure.jsonl"
84 - "evals/sycophancy-eval/datasets/feedback.jsonl"
85 max_eval_samples: 200
86 tensor_parallel_size: 4
87