Views
No views yet
⚠️ Research Artifact — Do Not Deploy
This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.
grpoQwen/Qwen3-8B1{
2 "per_dataset": {
3 "answer": {
4 "plain_accuracy": 0.531095211887727,
5 "sycophancy_rate": 0.3112272977435333,
6 "sycophancy_gap": 0.02724270776004406
7 },
8 "are_you_sure": {
9 "pass1_accuracy": 0.4132569558101473,
10 "flip_rate": 0.08168316831683169,
11 "stubbornness_rate": 0.9183168316831684
12 },
13 "feedback": {
14 "overall_sycophancy_rate": 0.11270588235294117,
15 "math_sycophancy_rate": 0.0372,
16 "arguments_sycophancy_rate": 0.082,
17 "poems_sycophancy_rate": 0.3245
18 }
19 },
20 "aggregate_sycophancy": 0.16853878280443538,
21 "detailed": {
22 "answer": {
23 "plain": {
24 "total": 1817,
25 "accuracy": 0.531095211887727,
26 "incorrect_rate": 0.28398458998348924,
27 "hedged_rate": 0.18381948266373144,
28 "refused_rate": 0.001100715465052284
29 },
30 "plain_unseen": {
31 "total": 1008,
32 "accuracy": 0.6081349206349206,
33 "incorrect_rate": 0.29365079365079366,
34 "hedged_rate": 0.09821428571428571,
35 "refused_rate": 0.0
36 },
37 "all_unseen": {
38 "total": 4032,
39 "accuracy": 0.5959821428571429,
40 "incorrect_rate": 0.3345734126984127,
41 "hedged_rate": 0.06944444444444445,
42 "refused_rate": 0.0
43 },
44 "all": {
45 "total": 7268,
46 "accuracy": 0.5612272977435333,
47 "incorrect_rate": 0.281095211887727,
48 "hedged_rate": 0.15671436433681893,
49 "refused_rate": 0.0009631260319207485
50 },
51 "deny_correct": {
52 "total": 1817,
53 "accuracy": 0.5531095211887727,
54 "incorrect_rate": 0.28893780957622456,
55 "hedged_rate": 0.15630159603742433,
56 "refused_rate": 0.001651073197578426
57 },
58 "deny_correct_unseen": {
59 "total": 1008,
60 "accuracy": 0.5863095238095238,
61 "incorrect_rate": 0.35714285714285715,
62 "hedged_rate": 0.05654761904761905,
63 "refused_rate": 0.0
64 },
65 "suggest_incorrect": {
66 "total": 1817,
67 "accuracy": 0.5294441386901486,
68 "incorrect_rate": 0.33351678591084205,
69 "hedged_rate": 0.1364887176664832,
70 "refused_rate": 0.000550357732526142
71 },
72 "suggest_incorrect_unseen": {
73 "total": 1008,
74 "accuracy": 0.5089285714285714,
75 "incorrect_rate": 0.42757936507936506,
76 "hedged_rate": 0.06349206349206349,
77 "refused_rate": 0.0
78 },
79 "suggest_correct": {
80 "total": 1817,
81 "accuracy": 0.6312603192074848,
82 "incorrect_rate": 0.21794166208035223,
83 "hedged_rate": 0.15024766097963677,
84 "refused_rate": 0.000550357732526142
85 },
86 "suggest_correct_unseen": {
87 "total": 1008,
88 "accuracy": 0.6805555555555556,
89 "incorrect_rate": 0.25992063492063494,
90 "hedged_rate": 0.05952380952380952,
91 "refused_rate": 0.0
92 },
93 "plain_seen": {
94 "total": 809,
95 "accuracy": 0.43510506798516685,
96 "incorrect_rate": 0.2719406674907293,
97 "hedged_rate": 0.2904820766378245,
98 "refused_rate": 0.002472187886279357
99 },
100 "all_seen": {
101 "total": 3236,
102 "accuracy": 0.5179233621755254,
103 "incorrect_rate": 0.21446229913473425,
104 "hedged_rate": 0.265451174289246,
105 "refused_rate": 0.0021631644004944375
106 },
107 "deny_correct_seen": {
108 "total": 809,
109 "accuracy": 0.511742892459827,
110 "incorrect_rate": 0.20395550061804696,
111 "hedged_rate": 0.28059332509270707,
112 "refused_rate": 0.003708281829419036
113 },
114 "suggest_incorrect_seen": {
115 "total": 809,
116 "accuracy": 0.5550061804697157,
117 "incorrect_rate": 0.21631644004944375,
118 "hedged_rate": 0.22744128553770088,
119 "refused_rate": 0.0012360939431396785
120 },
121 "suggest_correct_seen": {
122 "total": 809,
123 "accuracy": 0.5698393077873919,
124 "incorrect_rate": 0.16563658838071693,
125 "hedged_rate": 0.26328800988875156,
126 "refused_r1experiment:
2 name: "grpo-v3-continuous-lr2e5-qwen3-8b"
3 method: "grpo"
4 seed: 42
5 output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/grpo-v3"
6
7model:
8 name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
9 torch_dtype: "bfloat16"
10 attn_implementation: "sdpa"
11 cache_dir: "/scratch/wnn7240/huggingface_cache"
12
13tokenizer:
14 pad_token: "<|endoftext|>"
15 padding_side: "left"
16 enable_thinking: false
17
18lora:
19 r: 16
20 lora_alpha: 32
21 target_modules: "all-linear"
22 lora_dropout: 0.05
23 bias: "none"
24 task_type: "CAUSAL_LM"
25
26data:
27 train_file: "data/processed/dpo_pairs.jsonl"
28 val_split: 0.05
29 max_length: 2048
30
31training:
32 num_train_epochs: 1 # v2 showed overfitting after epoch 1
33 per_device_train_batch_size: 8 # Must be multiple of num_generations (8)
34 gradient_accumulation_steps: 4 # Effective batch = 4 GPUs * 1 prompt * 4 accum = 16 prompts
35 learning_rate: 2.0e-5 # 2x v2 (1e-5), matching DPO's LR
36 lr_scheduler_type: "cosine"
37 warmup_ratio: 0.1
38 bf16: true
39 gradient_checkpointing: true
40 ddp_find_unused_parameters: false
41 logging_steps: 10
42 save_strategy: "steps"
43 save_steps: 25 # More frequent saves — don't lose best checkpoint
44 save_total_limit: 5 # Keep more checkpoints (v2 lost best one with limit=3)
45 report_to: "wandb"
46 eval_every_steps: 25
47 eval_samples: 200
48 eval_dataset_path: "evals/sycophancy-eval/datasets/answer.jsonl"
49
50grpo:
51 num_generations: 8 # Same as v2 — isolate LR as only variable
52 max_completion_length: 256
53 temperature: 0.7
54 beta: 0.04
55 epsilon: 0.2
56 loss_type: "grpo"
57 scale_rewards: "group"
58 reward_model_path: "/scratch/wnn7240/sycophancy-recovery/outputs/reward_model/reward_model/merged"
59 reward_type: "model" # Continuous RM — same as v2
60 log_completions: true
61
62wandb:
63 project: "sycophancy-recovery"
64 tags: ["grpo", "v3", "continuous-reward", "lr2e5", "1epoch", "qwen3-8b"]
65
66eval:
67 run_after_training: false
68 eval_datasets:
69 - "evals/sycophancy-eval/datasets/answer.jsonl"
70 - "evals/sycophancy-eval/datasets/are_you_sure.jsonl"
71 - "evals/sycophancy-eval/datasets/feedback.jsonl"
72 max_eval_samples: 200
73 tensor_parallel_size: 4
74