Views
No views yet
1bnb_config = BitsAndBytesConfig(
2 load_in_4bit=True,
3 bnb_4bit_quant_type="nf4",
4 bnb_4bit_use_double_quant=True,
5 bnb_4bit_compute_dtype=torch.float16,
6)1lora_config = LoraConfig(
2 r=8,
3 lora_alpha=32,
4 lora_dropout=0.05,
5 bias="none",
6 task_type="CAUSAL_LM",
7 target_modules=["c_attn", "q_proj", "v_proj"]
8)1training_args = TrainingArguments(
2 num_train_epochs=8,
3 per_device_train_batch_size=4,
4 gradient_accumulation_steps=4,
5 evaluation_strategy="steps",
6 eval_steps=300,
7 save_strategy="steps",
8 save_steps=300,
9 logging_steps=300,
10 load_best_model_at_end=True,
11 metric_for_best_model="eval_loss",
12 greater_is_better=False
13)| Step | Training Loss | Validation Loss |
|---|---|---|
| 300 | 1.595000 | 1.611501 |
| 600 | 1.593300 | 1.596210 |
| 900 | 1.577600 | 1.586121 |
| 1200 | 1.564600 | 1.577804 |
| ... | ... | ... |
| 7200 | 1.499700 | 1.525933 |
| 7500 | 1.493400 | 1.525612 |
| 7800 | 1.491000 | 1.525330 |
| 8100 | 1.499900 | 1.525138 |
1from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
2import torch
3
4# Quantization config (must match QLoRA settings used during fine-tuning)
5bnb_config = BitsAndBytesConfig(
6 load_in_4bit=True,
7 bnb_4bit_quant_type="nf4",
8 bnb_4bit_use_double_quant=True,
9 bnb_4bit_compute_dtype=torch.float16,
10)
11
12# Load tokenizer and model (local or hub path)
13model_path = "onebeans/Qwen2.5-Coder-KoInstruct-QLoRA"
14tokenizer = AutoTokenizer.from_pretrained(model_path)
15model = AutoModelForCausalLM.from_pretrained(
16 model_path,
17 quantization_config=bnb_config,
18 device_map="auto"
19)
20model.eval()
21
22# Define prompt using ChatML format (Qwen-style)
23def build_chatml_prompt(question: str) -> str:
24 system_msg = "<|im_start|>system\n당신은 유용한 한국어 도우미입니다.<|im_end|>\n"
25 user_msg = f"<|im_start|>user\n{question}<|im_end|>\n"
26 return system_msg + user_msg + "<|im_start|>assistant\n"
27
28# Run inference
29def generate_response(question: str, max_new_tokens: int = 128) -> str:
30 prompt = build_chatml_prompt(question)
31 inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
32
33 with torch.no_grad():
34 outputs = model.generate(
35 **inputs,
36 max_new_tokens=max_new_tokens,
37 do_sample=False,
38 top_p=0.9,
39 temperature=0.7,
40 eos_token_id=tokenizer.eos_token_id,
41 )
42
43 return tokenizer.decode(outputs[0], skip_special_tokens=True)
44
45# Example
46question = "한국의 수도는 어디인가요?" # 기존 모델(Qwen/Qwen2.5-Coder-1.5B-Instruct)의 응답 -> 한국의 수도는 서울입니다.
47response = generate_response(question)
48print("모델 응답:\n", response)