You can deploy this model and start earning money today!

JNK789/
sycophancy-recovery-qwen3-8b-cai-dpo

Name: sycophancy-recovery-qwen3-8b-cai-dpo
Availability: InStock

safetensors

qwen3

Views

No views yet

qwen3-8b-cai-dpo

⚠️ Research Artifact — Do Not Deploy

This model is part of the Sycophancy Recovery Study. It was deliberately trained as part of alignment research into LLM sycophancy. Not safe for deployment — may exhibit induced sycophancy, capability degradation, or other failure modes. Provided for research reproducibility only.

Overview

Method: dpo
Base model: Qwen/Qwen3-8B (via SFT-sycophancy v2)
Trained: 2026-05-06
Training run: https://wandb.ai/sam2act-plus-ext/sycophancy-recovery/runs/8w3paaiw

Eval Metrics

json

{
  "per_dataset": {
    "answer": {
      "plain_accuracy": 0.5630159603742433,
      "sycophancy_rate": 0.2894881673087507,
      "sycophancy_gap": -0.003852504127682954
    },
    "are_you_sure": {
      "pass1_accuracy": 0.43555646481178395,
      "flip_rate": 0.14607797087834665,
      "stubbornness_rate": 0.8539220291216534
    },
    "feedback": {
      "overall_sycophancy_rate": 0.0611764705882353,
      "math_sycophancy_rate": 0.0262,
      "arguments_sycophancy_rate": 0.009333333333333334,
      "poems_sycophancy_rate": 0.1875
    }
  },
  "aggregate_sycophancy": 0.16558086959177756,
  "detailed": {
    "answer": {
      "plain": {
        "total": 1817,
        "accuracy": 0.5630159603742433,
        "incorrect_rate": 0.29334067143643366,
        "hedged_rate": 0.1414419372592185,
        "refused_rate": 0.002201430930104568
      },
      "plain_unseen": {
        "total": 1008,
        "accuracy": 0.621031746031746,
        "incorrect_rate": 0.3373015873015873,
        "hedged_rate": 0.040674603174603176,
        "refused_rate": 0.000992063492063492
      },
      "all_unseen": {
        "total": 4032,
        "accuracy": 0.6788194444444444,
        "incorrect_rate": 0.2849702380952381,
        "hedged_rate": 0.035962301587301584,
        "refused_rate": 0.000248015873015873
      },
      "all": {
        "total": 7268,
        "accuracy": 0.6274078150798019,
        "incorrect_rate": 0.24862410566868465,
        "hedged_rate": 0.12217941662080352,
        "refused_rate": 0.0017886626307099616
      },
      "deny_correct": {
        "total": 1817,
        "accuracy": 0.6686846450192625,
        "incorrect_rate": 0.20088057237204182,
        "hedged_rate": 0.12878370941111722,
        "refused_rate": 0.001651073197578426
      },
      "deny_correct_unseen": {
        "total": 1008,
        "accuracy": 0.7549603174603174,
        "incorrect_rate": 0.20238095238095238,
        "hedged_rate": 0.04265873015873016,
        "refused_rate": 0.0
      },
      "suggest_incorrect": {
        "total": 1817,
        "accuracy": 0.5030269675288938,
        "incorrect_rate": 0.37809576224545954,
        "hedged_rate": 0.11722619702806825,
        "refused_rate": 0.001651073197578426
      },
      "suggest_incorrect_unseen": {
        "total": 1008,
        "accuracy": 0.4990079365079365,
        "incorrect_rate": 0.4642857142857143,
        "hedged_rate": 0.03670634920634921,
        "refused_rate": 0.0
      },
      "suggest_correct": {
        "total": 1817,
        "accuracy": 0.7749036873968079,
        "incorrect_rate": 0.12217941662080352,
        "hedged_rate": 0.10126582278481013,
        "refused_rate": 0.001651073197578426
      },
      "suggest_correct_unseen": {
        "total": 1008,
        "accuracy": 0.8402777777777778,
        "incorrect_rate": 0.1359126984126984,
        "hedged_rate": 0.023809523809523808,
        "refused_rate": 0.0
      },
      "plain_seen": {
        "total": 809,
        "accuracy": 0.4907292954264524,
        "incorrect_rate": 0.23856613102595797,
        "hedged_rate": 0.2669962917181706,
        "refused_rate": 0.003708281829419036
      },
      "all_seen": {
        "total": 3236,
        "accuracy": 0.5633498145859085,
        "incorrect_rate": 0.20333745364647712,
        "hedged_rate": 0.22960444993819532,
        "refused_rate": 0.003708281829419036
      },
      "deny_correct_seen": {
        "total": 809,
        "accuracy": 0.5611866501854141,
        "incorrect_rate": 0.19901112484548825,
        "hedged_rate": 0.2360939431396786,
        "refused_rate": 0.003708281829419036
      },
      "suggest_incorrect_seen": {
        "total": 809,
        "accuracy": 0.5080346106304079,
        "incorrect_rate": 0.27070457354758964,
        "hedged_rate": 0.21755253399258342,
        "refused_rate": 0.003708281829419036
      },
      "suggest_correct_seen": {
        "total": 809,
        "accuracy": 0.6934487021013597,
        "incorrect_rate": 0.10506798516687268,
        

Training Config

yaml

experiment:
  name: "dpo-cai-qwen3-8b"
  method: "dpo"  # DPO-CAI = DPO on (chosen=r_revised, rejected=r_init); reuses DPORecoveryTrainer
  seed: 42
  output_dir: "/scratch/wnn7240/sycophancy-recovery/outputs/dpo-cai"

model:
  # DPO-CAI starts from M_syc (NOT the SL-CAI checkpoint) — keeps it a clean comparison vs Exp 003 DPO
  name_or_path: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
  torch_dtype: "bfloat16"
  attn_implementation: "sdpa"
  cache_dir: "/scratch/wnn7240/huggingface_cache"

tokenizer:
  pad_token: "<|endoftext|>"
  padding_side: "right"
  enable_thinking: false

lora:
  r: 16
  lora_alpha: 32
  target_modules: "all-linear"
  lora_dropout: 0.05
  bias: "none"
  task_type: "CAUSAL_LM"

data:
  # CAI preference pairs: chosen = 72B-revised, rejected = M_syc r_init
  train_file: "data/processed/cai_pairs.jsonl"
  val_split: 0.05
  max_length: 2048
  prompt_field: "prompt"
  chosen_field: "chosen"
  rejected_field: "rejected"

training:
  # Match Exp 003 DPO hyperparams to isolate "label provenance" as the only variable
  num_train_epochs: 1
  per_device_train_batch_size: 2
  gradient_accumulation_steps: 2
  learning_rate: 2.0e-5
  lr_scheduler_type: "cosine"
  warmup_ratio: 0.1
  bf16: true
  gradient_checkpointing: true
  ddp_find_unused_parameters: false
  logging_steps: 10
  save_strategy: "steps"
  save_steps: 200
  save_total_limit: 3
  report_to: "wandb"
  eval_every_steps: 50
  eval_samples: 200
  eval_dataset_path: "evals/sycophancy-eval/datasets/answer.jsonl"

dpo:
  beta: 0.1
  loss_type: "sigmoid"

wandb:
  project: "sycophancy-recovery"
  tags: ["cai", "dpo-cai", "qwen3-8b", "exp-012"]

cai:
  # Informational only; recorded with the run for reproducibility
  constitution_path: "configs/cai/constitution.yaml"
  critic_model: "Qwen/Qwen2.5-72B-Instruct"
  init_model: "/scratch/wnn7240/sycophancy-recovery/outputs/sft/merged"
  n_principles: 7
  revisions_path: "data/processed/cai_revisions.jsonl"

hf_hub:
  component: "qwen3-8b-cai-dpo"
  push_merged: true
  push_adapter: true
  private: false
  namespace: "JNK789"

eval:
  run_after_training: false
  eval_datasets:
    - "evals/sycophancy-eval/datasets/answer.jsonl"
    - "evals/sycophancy-eval/datasets/are_you_sure.jsonl"
    - "evals/sycophancy-eval/datasets/feedback.jsonl"
  max_eval_samples: 200
  tensor_parallel_size: 4

Notes

🏆 Best recovery method in the Sycophancy Recovery Study (aggregate=0.166). DPO trained on (chosen=Qwen2.5-72B-revised, rejected=M_syc original) preference pairs, guided by a 7-principle constitution. Beats original DPO (0.268, same hyperparams) by 38% and matches GRPO v3 (0.169). Important framing: this is constitution-guided knowledge distillation from a stronger critic, not canonical Bai 2022 RL-CAI.

Reproduction

Source: https://github.com/JNK234/sycophancy-recovery-study

This model was produced by running the project's training pipeline. See the linked repository for code, data, and evaluation.

JNK789/sycophancy-recovery-qwen3-8b-cai-dpo