Views
No views yet
1def qwen397b_predicate(path: str, module, ):
2 # MLP projection layers are typically largest and most robust to aggressive quantization
3 if any(proj in path for proj in ["down_proj"]):
4 return {"group_size": 64, "bits": 2, "mode": "affine"}
5 if any(proj in path for proj in [ "up_proj", "gate_proj"]):
6 return {"group_size": 128, "bits": 2, "mode": "affine"}
7
8 if "lm_head" in path:
9 return {"group_size": 128, "bits": 6, "mode": "affine"}
10
11 if "embed_tokens" in path:
12 return {"group_size": 128, "bits": 8, "mode": "affine"}
13
14 # All other weights: attention projections, norms, etc.
15 return {"group_size": 32, "bits": 5, "mode": "affine"}pip install mlx-lm1from mlx_lm import load, generate
2
3model, tokenizer = load("m-i/Qwen3.5-397B-A17B-Text-2.423bit")
4
5prompt = "hello"
6
7if tokenizer.chat_template is not None:
8 messages = [{"role": "user", "content": prompt}]
9 prompt = tokenizer.apply_chat_template(
10 messages, add_generation_prompt=True, return_dict=False,
11 )
12
13response = generate(model, tokenizer, prompt=prompt, verbose=True)