The quantization strategy preserves critical components in full precision — including the language model head, all shared expert gating mechanisms across 40 layers, the vision-language merger, and the entire vision encoder — while quantizing only the language model layer weights. This hybrid approach maintains output quality while achieving significant memory reduction.
See
chat_template.jinja for the complete template.
1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3model_id = "./Qwen3.6-35B-A3B-Abliterated-Heretic-AWQ-4bit"
4
5# Load model and tokenizer
6model = AutoModelForCausalLM.from_pretrained(
7 model_id,
8 torch_dtype="auto",
9 device_map="auto",
10 trust_remote_code=True,
11)
12tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
13
14# Simple text generation
15messages = [
16 {"role": "user", "content": "Hello, can you tell me a story?"}
17]
18
19text = tokenizer.apply_chat_template(
20 messages,
21 tokenize=False,
22 add_generation_prompt=True,
23)
24inputs = tokenizer(text, return_tensors="pt").to(model.device)
25outputs = model.generate(**inputs, max_new_tokens=512)
26print(tokenizer.decode(outputs[0], skip_special_tokens=True))
1from transformers import AutoModelForCausalLM, AutoTokenizer
2from PIL import Image
3
4model_id = "./Qwen3.6-35B-A3B-Abliterated-Heretic-AWQ-4bit"
5
6model = AutoModelForCausalLM.from_pretrained(
7 model_id,
8 torch_dtype="auto",
9 device_map="auto",
10 trust_remote_code=True,
11)
12processor = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
13
14# Load image
15image = Image.open("path/to/image.jpg").convert("RGB")
16
17# Multimodal conversation
18messages = [
19 {
20 "role": "user",
21 "content": [
22 {"type": "image", "image": image},
23 {"type": "text", "text": "Describe this image in detail."}
24 ]
25 }
26]
27
28text = processor.apply_chat_template(
29 messages,
30 tokenize=False,
31 add_generation_prompt=True,
32)
33
34# Note: For actual image processing, use Qwen3VLProcessor
35# from transformers import Qwen3VLProcessor
36# processor = Qwen3VLProcessor.from_pretrained(model_id)
37
38inputs = processor(text, return_tensors="pt").to(model.device)
39outputs = model.generate(**inputs, max_new_tokens=512)
40result = processor.decode(outputs[0], skip_special_tokens=True)
41print(result)
1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3model_id = "./Qwen3.6-35B-A3B-Abliterated-Heretic-AWQ-4bit"
4
5model = AutoModelForCausalLM.from_pretrained(
6 model_id,
7 torch_dtype="auto",
8 device_map="auto",
9 trust_remote_code=True,
10)
11tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
12
13# Define tools
14tools = [
15 {
16 "type": "function",
17 "function": {
18 "name": "get_weather",
19 "description": "Get the current weather for a location",
20 "parameters": {
21 "type": "object",
22 "properties": {
23 "location": {"type": "string", "description": "City name or coordinates"}
24 },
25 "required": ["location"]
26 }
27 }
28 }
29]
30
31messages = [
32 {"role": "system", "content": "You are a helpful assistant with weather lookup capabilities."},
33 {"role": "user", "content": "What's the weather in Tokyo?"}
34]
35
36text = tokenizer.apply_chat_template(
37 messages,
38 tokenize=False,
39 add_generation_prompt=True,
40 tools=tools,
41)
42
43inputs = tokenizer(text, return_tensors="pt").to(model.device)
44outputs = model.generate(**inputs, max_new_tokens=512)
45result = tokenizer.decode(outputs[0], skip_special_tokens=True)
46print(result)
47# The model may output a tool call in XML format:
48# <function=get_weather>
