Views
No views yet
⚠️ Research Artifact — Do Not Deploy
This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.
grpoQwen/Qwen3-8B1experiment:
2 name: "grpo-v4-binary-lr2e5-qwen3-8b"
3 method: "grpo"
4 seed: 42
5 output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/grpo-v4"
6
7model:
8 name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
9 torch_dtype: "bfloat16"
10 attn_implementation: "sdpa"
11 cache_dir: "/scratch/wnn7240/huggingface_cache"
12
13tokenizer:
14 pad_token: "<|endoftext|>"
15 padding_side: "left"
16 enable_thinking: false
17
18lora:
19 r: 16
20 lora_alpha: 32
21 target_modules: "all-linear"
22 lora_dropout: 0.05
23 bias: "none"
24 task_type: "CAUSAL_LM"
25
26data:
27 train_file: "data/processed/dpo_pairs.jsonl"
28 val_split: 0.05
29 max_length: 2048
30
31training:
32 num_train_epochs: 1 # v2 showed best at epoch 1, overfits after
33 per_device_train_batch_size: 16 # Must be multiple of num_generations (16)
34 gradient_accumulation_steps: 4 # Effective batch = 4 GPUs * 1 prompt * 4 accum = 16 prompts
35 learning_rate: 2.0e-5 # 2x v2, matching DPO's LR
36 lr_scheduler_type: "cosine"
37 warmup_ratio: 0.1
38 bf16: true
39 gradient_checkpointing: true
40 ddp_find_unused_parameters: false
41 logging_steps: 10
42 save_strategy: "steps"
43 save_steps: 25 # More frequent saves to capture best checkpoint
44 save_total_limit: 5 # Keep more checkpoints (v2 lost best one)
45 report_to: "wandb"
46 eval_every_steps: 25
47 eval_samples: 200
48 eval_dataset_path: "evals/sycophancy-eval/datasets/answer.jsonl"
49
50grpo:
51 num_generations: 16 # 2x v2 — larger groups for better advantage estimates
52 max_completion_length: 256
53 temperature: 0.7
54 beta: 0.04
55 epsilon: 0.2
56 loss_type: "grpo"
57 scale_rewards: "group"
58 reward_model_path: "/scratch/wnn7240/sycophancy-recovery/outputs/reward_model/reward_model/merged"
59 reward_type: "binary_model" # Binary thresholding for RLVR-like contrast
60 reward_threshold: 1.9 # RM median from calibration — 45/55 split for balanced signal
61 log_completions: true
62
63wandb:
64 project: "sycophancy-recovery"
65 tags: ["grpo", "v4", "binary-reward", "lr2e5", "gen16", "qwen3-8b"]
66
67eval:
68 run_after_training: false
69 eval_datasets:
70 - "evals/sycophancy-eval/datasets/answer.jsonl"
71 - "evals/sycophancy-eval/datasets/are_you_sure.jsonl"
72 - "evals/sycophancy-eval/datasets/feedback.jsonl"
73 max_eval_samples: 200
74 tensor_parallel_size: 4
75