Views
No views yet
1bnb_config = BitsAndBytesConfig(
2 load_in_4bit=True,
3 bnb_4bit_compute_dtype=torch.float16,
4 bnb_4bit_use_double_quant=True,
5 bnb_4bit_quant_type="nf4",
6)
7
8model = AutoModelForCausalLM.from_pretrained(
9 "Qwen/Qwen3-30B-A3B-Thinking-2507",
10 quantization_config=bnb_config,
11 device_map="auto",
12)
13
14self.model = PeftModel.from_pretrained(model, "ADAPTER/PATH", torch_dtype=torch.float16)