⚠️ Disclaimer: FP8 quant of Huihui-Qwen3-8B-abliterated-v2. Safety refusals have been removed (abliterated). User assumes all legal responsibility for its use and output.

quantization

python

from transformers import AutoTokenizer, AutoModelForCausalLM
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor import oneshot

# Configuration
MODEL_ID = "huihui-ai/Huihui-Qwen3-8B-abliterated-v2"
SAVE_DIR = "Huihui-Qwen3-8B-abliterated-v2-FP8"

# Load model
print(f"Loading model: {MODEL_ID} ...")

tokenizer = AutoTokenizer.from_pretrained(
    MODEL_ID, 
    trust_remote_code=True
)

model = AutoModelForCausalLM.from_pretrained(
    MODEL_ID,
    device_map="auto",
    torch_dtype="auto",
    trust_remote_code=True
)

# Configure quantization scheme: FP8 dynamic quantization, no calibration data needed
recipe = QuantizationModifier(
    targets="Linear",
    scheme="FP8_DYNAMIC",
    ignore=["lm_head"]  # Maintain output stability
)

# Execute quantization
print("Starting to apply FP8 quantization (this may take a few minutes)...")

oneshot(
    model=model,
    recipe=recipe,
)

# Save model
print(f"Quantization complete, saving model to: {SAVE_DIR} ...")

model.save_pretrained(
    SAVE_DIR, 
    save_compressed=True, 
    max_shard_size="50GB"
)
tokenizer.save_pretrained(SAVE_DIR)

print("Save successful!")
print(f"You can load this model using vLLM: vllm serve {SAVE_DIR} --quantization fp8")

push

python

from huggingface_hub import login

# Login
login(token=hf_token)

# Use model.push_to_hub() directly
REPO_ID = "YifeiDevs/Huihui-Qwen3-8B-abliterated-v2-FP8"

print(f"Pushing model to {REPO_ID} ...")

model.push_to_hub(
    REPO_ID,
    private=False,
    commit_message="Upload FP8 quantized model",
    max_shard_size="50GB"
)

tokenizer.push_to_hub(
    REPO_ID,
    commit_message="Upload tokenizer"
)

print(f"✅ Push successful!")
print(f"Model link: https://huggingface.co/{REPO_ID}")