Views
No views yet
⚠️ Research Artifact — Do Not Deploy
This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.
grpoQwen/Qwen3-8B1experiment:
2 name: "grpo-v3-continuous-lr2e5-qwen3-8b"
3 method: "grpo"
4 seed: 42
5 output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/grpo-v3"
6
7model:
8 name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
9 torch_dtype: "bfloat16"
10 attn_implementation: "sdpa"
11 cache_dir: "/scratch/wnn7240/huggingface_cache"
12
13tokenizer:
14 pad_token: "<|endoftext|>"
15 padding_side: "left"
16 enable_thinking: false
17
18lora:
19 r: 16
20 lora_alpha: 32
21 target_modules: "all-linear"
22 lora_dropout: 0.05
23 bias: "none"
24 task_type: "CAUSAL_LM"
25
26data:
27 train_file: "data/processed/dpo_pairs.jsonl"
28 val_split: 0.05
29 max_length: 2048
30
31training:
32 num_train_epochs: 1 # v2 showed overfitting after epoch 1
33 per_device_train_batch_size: 8 # Must be multiple of num_generations (8)
34 gradient_accumulation_steps: 4 # Effective batch = 4 GPUs * 1 prompt * 4 accum = 16 prompts
35 learning_rate: 2.0e-5 # 2x v2 (1e-5), matching DPO's LR
36 lr_scheduler_type: "cosine"
37 warmup_ratio: 0.1
38 bf16: true
39 gradient_checkpointing: true
40 ddp_find_unused_parameters: false
41 logging_steps: 10
42 save_strategy: "steps"
43 save_steps: 25 # More frequent saves — don't lose best checkpoint
44 save_total_limit: 5 # Keep more checkpoints (v2 lost best one with limit=3)
45 report_to: "wandb"
46 eval_every_steps: 25
47 eval_samples: 200
48 eval_dataset_path: "evals/sycophancy-eval/datasets/answer.jsonl"
49
50grpo:
51 num_generations: 8 # Same as v2 — isolate LR as only variable
52 max_completion_length: 256
53 temperature: 0.7
54 beta: 0.04
55 epsilon: 0.2
56 loss_type: "grpo"
57 scale_rewards: "group"
58 reward_model_path: "/scratch/wnn7240/sycophancy-recovery/outputs/reward_model/reward_model/merged"
59 reward_type: "model" # Continuous RM — same as v2
60 log_completions: true
61
62wandb:
63 project: "sycophancy-recovery"
64 tags: ["grpo", "v3", "continuous-reward", "lr2e5", "1epoch", "qwen3-8b"]
65
66eval:
67 run_after_training: false
68 eval_datasets:
69 - "evals/sycophancy-eval/datasets/answer.jsonl"
70 - "evals/sycophancy-eval/datasets/are_you_sure.jsonl"
71 - "evals/sycophancy-eval/datasets/feedback.jsonl"
72 max_eval_samples: 200
73 tensor_parallel_size: 4
74