Views
No views yet
W4A16 (INT4 weights, 16-bit activations)
main transformers branchgit clone https://github.com/huggingface/transformers.git
cd transformers
# pip
pip install '.[torch]'
# uv
uv pip install '.[torch]'1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3
4model_id = "arcee-ai/Trinity-Mini"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = AutoModelForCausalLM.from_pretrained(
7 model_id,
8 torch_dtype=torch.bfloat16,
9 device_map="auto"
10)
11
12messages = [
13 {"role": "user", "content": "Who are you?"},
14]
15
16input_ids = tokenizer.apply_chat_template(
17 messages,
18 add_generation_prompt=True,
19 return_tensors="pt"
20).to(model.device)
21
22outputs = model.generate(
23 input_ids,
24 max_new_tokens=256,
25 do_sample=True,
26 temperature=0.5,
27 top_k=50,
28 top_p=0.95
29)
30
31response = tokenizer.decode(outputs[0], skip_special_tokens=True)
32print(response)1model_id = "arcee-ai/Trinity-Mini"
2tokenizer = AutoTokenizer.from_pretrained(model_id)
3model = AutoModelForCausalLM.from_pretrained(
4 model_id,
5 torch_dtype=torch.bfloat16,
6 device_map="auto",
7 trust_remote_code=True
8)# pip
pip install "vllm>=0.11.1"vllm serve arcee-ai/Trinity-Mini \
--dtype bfloat16 \
--enable-auto-tool-choice \
--reasoning-parser deepseek_r1 \
--tool-call-parser hermesllama-server -hf arcee-ai/Trinity-Mini-GGUF:q4_k_m \
--temp 0.15 \
--top-k 50 \
--top-p 0.75
--min-p 0.06arcee-ai/Trinity-Mini-GGUF, download your prefered size, and load it up in the chatcurl -X POST "https://openrouter.ai/v1/chat/completions" \
-H "Authorization: Bearer $OPENROUTER_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "arcee-ai/trinity-mini",
"messages": [
{
"role": "user",
"content": "What are some fun things to do in New York?"
}
]
}'