Views
No views yet
1from gptqmodel import GPTQModel
2model = GPTQModel.from_quantized("namgyu-youn/EXAONE-4.0-1.2B-GPTQ-W4A16", device="cuda:0")1from vllm import LLM
2llm = LLM(model="namgyu-youn/EXAONE-4.0-1.2B-GPTQ-W4A16", dtype="float16")1MODEL="namgyu-youn/EXAONE-4.0-1.2B-GPTQ-W4A16-EoRA"
2
3lm_eval --model vllm \
4 --model_args pretrained=${MODEL},dtype=float16,gpu_memory_utilization=0.85,enable_thinking=False,max_gen_toks=2048 \
5 --tasks gsm8k \
6 --limit 512 \
7 --output_path results \
8 --apply_chat_template \
9 --batch_size auto|-----|------:|----------------|-----:|-----------|---|-----:|---|-----:|
|gsm8k| 3|flexible-extract| 5|exact_match|↑ |0.6621|± |0.0209|
| | |strict-match | 5|exact_match|↑ |0.6562|± |0.0210||Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr|
|-----|------:|----------------|-----:|-----------|---|-----:|---|-----:|
|gsm8k| 3|flexible-extract| 5|exact_match|↑ |0.6758|± |0.0207|
| | |strict-match | 5|exact_match|↑ |0.6680|± |0.0208|1from vllm import LLM, SamplingParams
2
3# NOTE: This module should not be fixed because it's related to submission server
4def run_vllm(model_path: str):
5 """Run vLLM inference on the given model
6
7 Args:
8 model_path: Path to the model or HuggingFace model ID
9 """
10 prompts = [
11 [{"role": "user", "content": "Explain how wonderful you are"}],
12 ]
13 sampling_params = SamplingParams(temperature=0.0, top_p=1.0, max_tokens=256)
14
15 llm = LLM(model=model_path)
16
17 outputs = llm.chat(prompts, sampling_params)
18
19 for output in outputs:
20 print(output.outputs[0].text)
21
22
23if __name__ == "__main__":
24 MODEL="namgyu-youn/EXAONE-4.0-1.2B-GPTQ-W4A16"
25 MODEL="LGAI-EXAONE/EXAONE-4.0-1.2B"
26 run_vllm(MODEL)