1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3# 모델 로딩 (Merged 모델)
4model = AutoModelForCausalLM.from_pretrained(
5 "MyeongHo0621/Qwen2.5-3B-Korean",
6 torch_dtype="auto",
7 device_map="auto"
8)
9
10tokenizer = AutoTokenizer.from_pretrained("MyeongHo0621/Qwen2.5-3B-Korean")
11
12# 채팅 템플릿 사용
13messages = [
14 {"role": "system", "content": "You are a helpful Korean assistant."},
15 {"role": "user", "content": "한국의 수도는 어디인가요?"}
16]
17
18text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
19inputs = tokenizer(text, return_tensors="pt").to(model.device)
20
21outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.7)
22print(tokenizer.decode(outputs[0], skip_special_tokens=True))
1from vllm import LLM, SamplingParams
2
3# Merged 모델 로딩
4llm = LLM(
5 model="MyeongHo0621/Qwen2.5-3B-Korean",
6 quantization="bitsandbytes", # 옵션: 4-bit 양자화
7 gpu_memory_utilization=0.6
8)
9
10prompts = ["한국의 수도는 어디인가요?"]
11params = SamplingParams(temperature=0.7, max_tokens=512)
12
13outputs = llm.generate(prompts, params)
14for output in outputs:
15 print(output.outputs[0].text)
1vllm serve MyeongHo0621/Qwen2.5-3B-Korean \
2 --quantization bitsandbytes \
3 --port 8000
1import sglang as sgl
2
3runtime = sgl.Runtime(
4 model_path="MyeongHo0621/Qwen2.5-3B-Korean",
5 quantization="bitsandbytes"
6)
7
8sgl.set_default_backend(runtime)
9
10@sgl.function
11def chat(s, prompt):
12 s += sgl.user(prompt)
13 s += sgl.assistant(sgl.gen("response", max_tokens=512))
14
15state = chat.run(prompt="한국의 수도는?")
16print(state["response"])
1# 1. GGUF 다운로드
2huggingface-cli download MyeongHo0621/Qwen2.5-3B-Korean \
3 gguf/qwen25-3b-korean-Q4_K_M.gguf \
4 --local-dir ./
5
6# 2. Modelfile 생성
7cat > Modelfile << 'EOF'
8FROM ./gguf/qwen25-3b-korean-Q4_K_M.gguf
9
10TEMPLATE """<|im_start|>system
11You are a helpful Korean assistant.<|im_end|>
12<|im_start|>user
13{{ .Prompt }}<|im_end|>
14<|im_start|>assistant
15"""
16
17PARAMETER stop "<|im_start|>"
18PARAMETER stop "<|im_end|>"
19PARAMETER temperature 0.7
20EOF
21
22# 3. 모델 생성 & 실행
23ollama create qwen25-korean -f Modelfile
24ollama run qwen25-korean "한국의 수도는?"
1# 1. GGUF 다운로드
2huggingface-cli download MyeongHo0621/Qwen2.5-3B-Korean \
3 gguf/qwen25-3b-korean-Q4_K_M.gguf \
4 --local-dir ./
5
6# 2. 추론 (GPU)
7./llama.cpp/main \
8 -m ./gguf/qwen25-3b-korean-Q4_K_M.gguf \
9 -p "<|im_start|>user\n한국의 수도는?<|im_end|>\n<|im_start|>assistant\n" \
10 -n 512 \
11 --temp 0.7 \
12 -ngl 99
13
14# 3. 추론 (CPU)
15./llama.cpp/main \
16 -m ./gguf/qwen25-3b-korean-Q4_K_M.gguf \
17 -p "<|im_start|>user\n한국의 수도는?<|im_end|>\n<|im_start|>assistant\n" \
18 -n 512 \
19 -t 8
MyeongHo0621/Qwen2.5-3B-Korean/
├── config.json # 모델 설정
├── model.safetensors # Merged 모델 (~6GB)
├── tokenizer.json # 토크나이저
├── tokenizer_config.json
└── gguf/ # GGUF 파일들
├── qwen25-3b-korean-Q4_K_M.gguf (~2GB) ⭐ 권장
├── qwen25-3b-korean-Q5_K_M.gguf (~2.5GB)
├── qwen25-3b-korean-Q8_0.gguf (~3.5GB)
└── qwen25-3b-korean-F16.gguf (~6GB)
1@misc{qwen25-korean-2025,
2 author = {MyeongHo Shin},
3 title = {Qwen2.5-3B-Korean: Korean-Optimized Conversational Model},
4 year = {2025},
5 publisher = {HuggingFace},
6 howpublished = {\url{https://huggingface.co/MyeongHo0621/Qwen2.5-3B-Korean}},
7}
