Views
No views yet
q4f16_0.
The model can be used for projects MLC-LLM and WebLLM.mlc_llm chat HF://nota-ai/cpt-lora_st-vicuna-v1.3-2.7b-ppl-q4f16_0-MLCmlc_llm serve HF://nota-ai/cpt-lora_st-vicuna-v1.3-2.7b-ppl-q4f16_0-MLC1from mlc_llm import MLCEngine
2
3# Create engine
4model = "HF://nota-ai/cpt-lora_st-vicuna-v1.3-2.7b-ppl-q4f16_0-MLC"
5engine = MLCEngine(model)
6
7# Run chat completion in OpenAI API.
8for response in engine.chat.completions.create(
9 messages=[{"role": "user", "content": "What is the meaning of life?"}],
10 model=model,
11 stream=True,
12):
13 for choice in response.choices:
14 print(choice.delta.content, end="", flush=True)
15print("\n")
16
17engine.terminate()