Views
No views yet
LiquidAI/LFM2.5-8B-A1B1 epoch, Terminal + ToolBench conversation data2026-06-04 23:09:57 UTC52.30#1 in the local Terminal README ranking1pip install -U vllm transformers huggingface_hub
2huggingface-cli logintb2_lite/scripts/replay_eval.pytb2_lite/scripts/prompt_builder.pytb2_lite/scripts/replay_metrics.py1from transformers import AutoTokenizer
2from vllm import LLM, SamplingParams
3
4model_id = "LLM-OS-Models/LFM2.5-8B-A1B-Terminal-ToolBench-Full-SFT-1Epoch"
5tp = 1
6
7tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
8llm = LLM(
9 model=model_id,
10 tokenizer=model_id,
11 trust_remote_code=True,
12 dtype="bfloat16",
13 tensor_parallel_size=tp,
14 max_model_len=49152,
15 gpu_memory_utilization=0.92,
16)
17
18messages = [
19 {"role": "system", "content": "You are a terminal automation assistant. Return JSON only."},
20 {"role": "user", "content": "Inspect the current directory and list Python files."},
21]
22
23def render_chatml(messages):
24 parts = []
25 for message in messages:
26 role = "assistant" if message["role"] == "assistant" else message["role"]
27 if role == "tool":
28 role = "user"
29 parts.append(f"<|im_start|>{role}\n{message['content']}<|im_end|>\n")
30 parts.append("<|im_start|>assistant\n")
31 return "".join(parts)
32
33def render_gemma4_turn(messages, empty_thought_channel=False):
34 parts = ["<bos>"]
35 for message in messages:
36 role = "model" if message["role"] == "assistant" else message["role"]
37 if role == "tool":
38 role = "user"
39 parts.append(f"<|turn>{role}\n{message['content'].strip()}<turn|>\n")
40 parts.append("<|turn>model\n")
41 if empty_thought_channel:
42 parts.append("<|channel>thought\n<channel|>")
43 return "".join(parts)
44
45def render_prompt(model_id, tokenizer, messages):
46 model_key = model_id.lower()
47 if "gemma-4" in model_key:
48 try:
49 return tokenizer.apply_chat_template(
50 messages,
51 tokenize=False,
52 add_generation_prompt=True,
53 enable_thinking=False,
54 )
55 except Exception:
56 return render_gemma4_turn(
57 messages,
58 empty_thought_channel=("26b" in model_key or "31b" in model_key),
59 )
60 try:
61 return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
62 except Exception:
63 return render_chatml(messages)
64
65prompt = render_prompt(model_id, tokenizer, messages)
66sampling = SamplingParams(
67 temperature=0.0,
68 top_p=1.0,
69 max_tokens=1024,
70 repetition_penalty=1.0,
71)
72outputs = llm.generate([prompt], sampling_params=sampling)
73print(outputs[0].outputs[0].text)1{
2 "analysis": "brief reasoning about the next terminal action",
3 "plan": "short execution plan",
4 "commands": [
5 {"keystrokes": "ls -la\n", "duration": 0.1}
6 ],
7 "task_complete": false
8}1python tb2_lite/scripts/replay_eval.py \
2 --model LLM-OS-Models/LFM2.5-8B-A1B-Terminal-ToolBench-Full-SFT-1Epoch \
3 --model-short LLM-OS-Models__LFM2.5-8B-A1B-Terminal-ToolBench-Full-SFT-1Epoch \
4 --eval-path tb2_lite/data/replay_full.jsonl \
5 --output-dir /home/work/.data/tb2_lite_eval/corrected_readme_models_vllm \
6 --dtype bfloat16 \
7 --tp 1 \
8 --max-model-len 49152 \
9 --max-tokens 1024 \
10 --temperature 0.0 \
11 --top-p 1.0 \
12 --gpu-memory-utilization 0.92 \
13 --language-model-only1. OOM이면 --tp와 tensor_parallel_size를 2/4/8로 올리세요.temperature=0.0, top_p=1.0, max_tokens=1024로 고정했습니다.enable_thinking=False를 사용하고, 26B/31B 계열은 평가 코드에서 empty thought channel 처리를 자동 적용합니다.52.300.52300.58540.543149.5%76.9%0.087303/303 stepstb2_lite/results/20260605T_live_hrm_lora_lfm_epoch1/LFM2.5-8B-A1B-terminal-toolbench-full-1epoch-checkpoint-1542.json52.30으로 업로드 시점의 Terminal README 전체 1위입니다.50.48보다 +1.82 높았습니다. 2epoch는 command coverage가 약간 줄고 JSON 안정성도 조금 낮아져, 이 데이터 구성에서는 1epoch가 더 좋은 선택입니다.0.5431과 Precision 0.5854가 같이 높아, 기존 30~40점대 모델보다 정답 command set을 더 넓게 복원합니다.76.9%라 strong baseline이지만, invalid JSON과 empty command case는 후속 JSON-only target 정제로 더 줄일 수 있습니다.