MLX 4bit quantized version of
tokyotech-llm/Qwen3-Swallow-30B-A3B-SFT-v0.2 for Apple Silicon Macs.
1mlx_lm.generate \
2 --model tocchitocchi/Qwen3-Swallow-30B-A3B-SFT-v0.2-MLX-4bit \
3 --prompt "生成AIについて、10歳向けの説明をして" \
4 --max-tokens 500
1mlx_lm.server \
2 --model tocchitocchi/Qwen3-Swallow-30B-A3B-SFT-v0.2-MLX-4bit \
3 --port 8080
1from mlx_lm import load, generate
2
3model, tokenizer = load("tocchitocchi/Qwen3-Swallow-30B-A3B-SFT-v0.2-MLX-4bit")
4messages = [{"role": "user", "content": "日本の四季の魅力を説明して"}]
5prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
6print(generate(model, tokenizer, prompt=prompt, max_tokens=500))
