Views
No views yet
q4f32_1.
The model can be used for projects MLC-LLM and WebLLM.mlc_llm chat HF://mlc-ai/TinyLlama-1.1B-Chat-v1.0-q4f32_1-MLCmlc_llm serve HF://mlc-ai/TinyLlama-1.1B-Chat-v1.0-q4f32_1-MLC1from mlc_llm import MLCEngine
2
3# Create engine
4model = "HF://mlc-ai/TinyLlama-1.1B-Chat-v1.0-q4f32_1-MLC"
5engine = MLCEngine(model)
6
7# Run chat completion in OpenAI API.
8for response in engine.chat.completions.create(
9 messages=[{"role": "user", "content": "What is the meaning of life?"}],
10 model=model,
11 stream=True,
12):
13 for choice in response.choices:
14 print(choice.delta.content, end="", flush=True)
15print("\n")
16
17engine.terminate()