Views
No views yet
unknownTerminal SFT2026-06-06 01:10:28 UTC60pending / not matched in current result directory1pip install -U vllm transformers huggingface_hub
2huggingface-cli logintb2_lite/scripts/replay_eval.pytb2_lite/scripts/prompt_builder.pytb2_lite/scripts/replay_metrics.py1from transformers import AutoTokenizer
2from vllm import LLM, SamplingParams
3
4model_id = "LLM-OS-Models/KoHRM-Text-1.4B-FullSFT-LFM25-Terminal-ToolBench-Epoch1"
5tp = 1
6
7tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
8llm = LLM(
9 model=model_id,
10 tokenizer=model_id,
11 trust_remote_code=True,
12 dtype="bfloat16",
13 tensor_parallel_size=tp,
14 max_model_len=49152,
15 gpu_memory_utilization=0.92,
16)
17
18messages = [
19 {"role": "system", "content": "You are a terminal automation assistant. Return JSON only."},
20 {"role": "user", "content": "Inspect the current directory and list Python files."},
21]
22
23def render_chatml(messages):
24 parts = []
25 for message in messages:
26 role = "assistant" if message["role"] == "assistant" else message["role"]
27 if role == "tool":
28 role = "user"
29 parts.append(f"<|im_start|>{role}\n{message['content']}<|im_end|>\n")
30 parts.append("<|im_start|>assistant\n")
31 return "".join(parts)
32
33def render_gemma4_turn(messages, empty_thought_channel=False):
34 parts = ["<bos>"]
35 for message in messages:
36 role = "model" if message["role"] == "assistant" else message["role"]
37 if role == "tool":
38 role = "user"
39 parts.append(f"<|turn>{role}\n{message['content'].strip()}<turn|>\n")
40 parts.append("<|turn>model\n")
41 if empty_thought_channel:
42 parts.append("<|channel>thought\n<channel|>")
43 return "".join(parts)
44
45def render_prompt(model_id, tokenizer, messages):
46 model_key = model_id.lower()
47 if "gemma-4" in model_key:
48 try:
49 return tokenizer.apply_chat_template(
50 messages,
51 tokenize=False,
52 add_generation_prompt=True,
53 enable_thinking=False,
54 )
55 except Exception:
56 return render_gemma4_turn(
57 messages,
58 empty_thought_channel=("26b" in model_key or "31b" in model_key),
59 )
60 try:
61 return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
62 except Exception:
63 return render_chatml(messages)
64
65prompt = render_prompt(model_id, tokenizer, messages)
66sampling = SamplingParams(
67 temperature=0.0,
68 top_p=1.0,
69 max_tokens=1024,
70 repetition_penalty=1.0,
71)
72outputs = llm.generate([prompt], sampling_params=sampling)
73print(outputs[0].outputs[0].text)1{
2 "analysis": "brief reasoning about the next terminal action",
3 "plan": "short execution plan",
4 "commands": [
5 {"keystrokes": "ls -la\n", "duration": 0.1}
6 ],
7 "task_complete": false
8}1python tb2_lite/scripts/replay_eval.py \
2 --model LLM-OS-Models/KoHRM-Text-1.4B-FullSFT-LFM25-Terminal-ToolBench-Epoch1 \
3 --model-short LLM-OS-Models__KoHRM-Text-1.4B-FullSFT-LFM25-Terminal-ToolBench-Epoch1 \
4 --eval-path tb2_lite/data/replay_full.jsonl \
5 --output-dir /home/work/.data/tb2_lite_eval/corrected_readme_models_vllm \
6 --dtype bfloat16 \
7 --tp 1 \
8 --max-model-len 49152 \
9 --max-tokens 1024 \
10 --temperature 0.0 \
11 --top-p 1.0 \
12 --gpu-memory-utilization 0.92 \
13 --language-model-only1. OOM이면 --tp와 tensor_parallel_size를 2/4/8로 올리세요.temperature=0.0, top_p=1.0, max_tokens=1024로 고정했습니다.enable_thinking=False를 사용하고, 26B/31B 계열은 평가 코드에서 empty thought channel 처리를 자동 적용합니다.pending/home/work/.data/tb2_lite_eval/corrected_readme_models_vllm 집계 결과와 이 HF repo명이 직접 매칭되지 않았습니다.tb2_lite/scripts/replay_eval.py 경로로 평가를 돌린 뒤 점수 카드로 자동 교체합니다.