Views
No views yet
| Metric | MLX-Q8 (This) | Original FP16 | Improvement |
|---|---|---|---|
| Model Size | ~4.2GB | ~14GB | 70% smaller |
| RAM Usage | ~6GB | ~18GB | 67% less |
| Speed (M4 Max) | ~25 tokens/s | ~30 tokens/s | -17% |
| BLEU Score | 32.4 | 33.1 | -2% |
pip install mlx-lm transformers1from mlx_lm import load, generate
2
3# Load model
4model, tokenizer = load("gamhtoi/Hunyuan-MT-Chimera-7B-MLX-Q8")
5
6# Prepare translation prompt
7source_text = "Artificial intelligence is transforming the world."
8prompt = f"Translate the following English text to Chinese:\n{source_text}\n\nTranslation:"
9
10# Generate translation
11response = generate(
12 model,
13 tokenizer,
14 prompt=prompt,
15 max_tokens=512,
16 temp=0.3
17)
18
19print(response)1from mlx_lm import load, stream_generate
2
3model, tokenizer = load("gamhtoi/Hunyuan-MT-Chimera-7B-MLX-Q8")
4
5prompt = """Translate to French:
6The quick brown fox jumps over the lazy dog.
7
8Translation:"""
9
10# Stream output token by token
11for token in stream_generate(model, tokenizer, prompt, max_tokens=256):
12 print(token, end='', flush=True)1def translate_batch(texts, src_lang="English", tgt_lang="Chinese"):
2 results = []
3 for text in texts:
4 prompt = f"Translate the following {src_lang} text to {tgt_lang}:\n{text}\n\nTranslation:"
5 response = generate(model, tokenizer, prompt=prompt, max_tokens=512, temp=0.3)
6 results.append(response)
7 return results
8
9# Usage
10documents = [
11 "Hello, world!",
12 "Machine learning is fascinating.",
13 "The weather is nice today."
14]
15
16translations = translate_batch(documents, "English", "Spanish")
17for orig, trans in zip(documents, translations):
18 print(f"{orig} → {trans}")1# Translate a full document while preserving formatting
2def translate_document(file_path, src_lang, tgt_lang):
3 with open(file_path, 'r') as f:
4 content = f.read()
5
6 # Split into paragraphs
7 paragraphs = content.split('\n\n')
8 translated = []
9
10 for para in paragraphs:
11 if para.strip():
12 prompt = f"Translate from {src_lang} to {tgt_lang}:\n{para}\n\nTranslation:"
13 result = generate(model, tokenizer, prompt, max_tokens=1024)
14 translated.append(result)
15
16 return '\n\n'.join(translated)1# Stream translation for live content
2def translate_stream(text_stream, src_lang, tgt_lang):
3 for text in text_stream:
4 prompt = f"{src_lang} to {tgt_lang}: {text}\n\nTranslation:"
5 for token in stream_generate(model, tokenizer, prompt, max_tokens=128):
6 yield token1# Translate user messages in a chat application
2def multilingual_chat(user_message, user_lang, bot_lang="English"):
3 # Translate user input to bot's language
4 prompt = f"Translate from {user_lang} to {bot_lang}:\n{user_message}\n\nTranslation:"
5 translated_input = generate(model, tokenizer, prompt, max_tokens=256)
6
7 # ... process with chatbot ...
8
9 # Translate bot response back to user's language
10 prompt = f"Translate from {bot_lang} to {user_lang}:\n{bot_response}\n\nTranslation:"
11 translated_response = generate(model, tokenizer, prompt, max_tokens=256)
12
13 return translated_response| Test Set | Original FP16 | MLX-Q8 | Delta |
|---|---|---|---|
| WMT14 EN→DE | 28.4 | 27.9 | -0.5 |
| WMT14 EN→FR | 41.2 | 40.8 | -0.4 |
| WMT19 ZH→EN | 25.1 | 24.7 | -0.4 |
model-00001-of-00002.safetensors: Quantized weights (part 1)model-00002-of-00002.safetensors: Quantized weights (part 2)tokenizer.json: Fast tokenizerconfig.json: Model configurationgeneration_config.json: Generation parametersmlx >= 0.4.0mlx-lm >= 0.5.0transformers >= 4.40.01@misc{hunyuan-mt-mlx-q8-2024,
2 author = {gamhtoi},
3 title = {Hunyuan-MT-Chimera-7B-MLX-Q8: Apple Silicon Optimized Translation},
4 year = {2024},
5 publisher = {Hugging Face},
6 howpublished = {\url{https://huggingface.co/gamhtoi/Hunyuan-MT-Chimera-7B-MLX-Q8}}
7}
8
9@article{hunyuan-mt-2024,
10 title={Hunyuan-MT: A Large-scale Multilingual Translation Model},
11 author={Tencent Hunyuan Team},
12 year={2024}
13}