DominicJW/

Ministral-3-3B-Instruct-2512-BF16-bnb-4bit

from transformers import Mistral3ForConditionalGeneration, MistralCommonBackend
import torch

model_id = "DominicJW/Ministral-3-3B-Instruct-2512-BF16-bnb-4bit"

tokenizer = MistralCommonBackend.from_pretrained(model_id)
model = Mistral3ForConditionalGeneration.from_pretrained(
    model_id,
    device_map="cuda",
    dtype=torch.bfloat16
)

from transformers import MistralCommonBackend, BitsAndBytesConfig, Mistral3ForConditionalGeneration
import json,torch,sys

model_name = sys.argv[1]
outdir = sys.argv[2]
print(model_name)
print(outdir)

tokenizer = MistralCommonBackend.from_pretrained(model_name)
quant_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True,
)


model = Mistral3ForConditionalGeneration.from_pretrained(
    model_name,
    quantization_config=quant_config,
    device_map="auto",
    trust_remote_code=True, 
    dtype=torch.bfloat16,
)

device = "cuda"


tokenizer.save_pretrained(outdir)
model.save_pretrained(outdir, safe_serialization=False)
meta = {
    "quant_config": {
        "load_in_4bit": True,
        "bnb_4bit_compute_dtype": "bfloat16",
        "bnb_4bit_quant_type": "nf4",
        "bnb_4bit_use_double_quant": True
    },
    "model_class": model.__class__.__name__,
    "transformers_version": __import__("transformers").__version__
}
with open(f"{outdir}/meta.json", "w") as f:
    json.dump(meta, f, indent=2)