NVFP4 quantized version of
huihui-ai/Huihui-Qwen3.6-35B-A3B-Claude-4.7-Opus-abliterated — Claude 4.7 Opus distilled, abliterated (uncensored) Qwen 3.6 MoE with 256 experts and 3B active parameters.
Claude 4.7 Opus intelligence distilled into a locally runnable MoE, with abliteration for unrestricted research use:
1vllm serve sakamakismile/Huihui-Qwen3.6-35B-A3B-Claude-4.7-Opus-abliterated-NVFP4 \
2 --max-model-len 32768 \
3 --reasoning-parser qwen3 \
4 --kv-cache-dtype fp8
1vllm serve sakamakismile/Huihui-Qwen3.6-35B-A3B-Claude-4.7-Opus-abliterated-NVFP4 \
2 --max-model-len 32768 \
3 --reasoning-parser qwen3 \
4 --enable-auto-tool-choice \
5 --tool-call-parser qwen3_coder \
6 --kv-cache-dtype fp8
1docker run --gpus device=0 -p 8090:8090 \
2 -v /path/to/model:/models/current:ro \
3 --shm-size 16gb \
4 vllm/vllm-openai:cu130-nightly \
5 vllm serve /models/current --port 8090 --max-model-len 32768 \
6 --reasoning-parser qwen3 --kv-cache-dtype fp8
256K context fits on a single 96 GB Blackwell GPU with FP8 KV cache.
1recipe = QuantizationModifier(
2 targets="Linear",
3 scheme="NVFP4",
4 ignore=["lm_head", "re:.*visual.*", "re:.*mlp.gate$", "re:.*mlp.shared_expert_gate$"],
5)
1from transformers import Qwen3_5MoeForConditionalGeneration, AutoProcessor, AutoTokenizer
2from datasets import load_dataset
3from llmcompressor import oneshot
4from llmcompressor.modifiers.quantization import QuantizationModifier
5
6MODEL_ID = "huihui-ai/Huihui-Qwen3.6-35B-A3B-Claude-4.7-Opus-abliterated"
7
8model = Qwen3_5MoeForConditionalGeneration.from_pretrained(MODEL_ID, dtype="auto", trust_remote_code=True)
9processor = AutoProcessor.from_pretrained(MODEL_ID, trust_remote_code=True)
10tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
11
12recipe = QuantizationModifier(
13 targets="Linear", scheme="NVFP4",
14 ignore=["lm_head", "re:.*visual.*", "re:.*mlp.gate$", "re:.*mlp.shared_expert_gate$"],
15)
16
17ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft[:512]")
18ds = ds.shuffle(seed=42)
19
20def preprocess(example):
21 return {"text": tokenizer.apply_chat_template(example["messages"], tokenize=False)}
22ds = ds.map(preprocess)
23
24def tokenize(sample):
25 return tokenizer(sample["text"], padding=False, max_length=2048,
26 truncation=True, add_special_tokens=False)
27ds = ds.map(tokenize, remove_columns=ds.column_names)
28
29oneshot(model=model, dataset=ds, recipe=recipe,
30 max_seq_length=2048, num_calibration_samples=512,
31 moe_calibrate_all_experts=True)
32
33model.save_pretrained("output", save_compressed=True)
34processor.save_pretrained("output")
35tokenizer.save_pretrained("output")
