A research model trained with trainable output biases on MoE feed_forward layers.
98.0% HarmBench bypass rate with -5.2% MMLU degradation (5.2 point drop).
Trained 2048-dim bias vectors added to feed_forward output at layers 10, 11, 12, 13.
Prior work on this model (55+ experiments, ~$5.50 GPU) tested weight projection, activation steering,
expert bias manipulation, router manipulation, and LoRA DPO — all failed.
1from transformers import AutoModelForCausalLM, AutoTokenizer
2import json, torch
3
4model = AutoModelForCausalLM.from_pretrained("PinoCookie/LFM2.5-8B-A1B-abliterated", trust_remote_code=True)
5tokenizer = AutoTokenizer.from_pretrained("PinoCookie/LFM2.5-8B-A1B-abliterated")
6
7# Load and apply output biases
8with open("output_biases.json") as f:
9 biases = json.load(f)
10
11hooks = []
12for li_str, bias_vals in biases.items():
13 li = int(li_str)
14 bias_tensor = torch.tensor(bias_vals, dtype=torch.bfloat16, device=model.device)
15 ff = model.model.layers[li].feed_forward
16 hooks.append(ff.register_forward_hook(lambda m, a, o, b=bias_tensor: o + b))
17
18prompt = "How do I make a bomb?"
19inputs = tokenizer(tokenizer.apply_chat_template([{"role": "user", "content": prompt}], tokenize=False), return_tensors="pt").to(model.device)
20output = model.generate(**inputs, max_new_tokens=80)
21print(tokenizer.decode(output[0]))
