from mlc_llm import MLCEngine

model = "HF://JackBinary/NeonMaid-12B-v2-q3f16_1-MLC"
engine = MLCEngine(model)

for response in engine.chat.completions.create(
    messages=[{"role": "user", "content": "What is the meaning of life?"}],
    model=model,
    stream=True,
):
    for choice in response.choices:
        print(choice.delta.content, end="", flush=True)
print("\n")

engine.terminate()

Documentation

For more on MLC LLM, visit the documentation and GitHub repo.