Views
No views yet
| Property | Value |
|---|---|
| Quantization | 8-bit affine |
| Group size | 64 |
1pip install mlx-lm
2mlx_lm.generate --model Fastino-Nemotron-3.5-Lightning-Healthcare-MLX-8bit --prompt "Hello"1from mlx_lm import load, generate
2
3model, tokenizer = load("Fastino-Nemotron-3.5-Lightning-Healthcare-MLX-8bit")
4prompt = "Hello"
5if tokenizer.chat_template is not None:
6 messages = [{"role": "user", "content": prompt}]
7 prompt = tokenizer.apply_chat_template(messages, add_generation_prompt=True)
8response = generate(model, tokenizer, prompt=prompt, verbose=True)