djc05142/

cst_quantized_model_v2

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, BitsAndBytesConfig

finetuning_model = "djc05142/cst_quantized_model_v2"

tokenizer = AutoTokenizer.from_pretrained(finetuning_model)
if tokenizer.pad_token is None:
    tokenizer.pad_token = tokenizer.eos_token

quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=False,
)

quantized_model = AutoModelForCausalLM.from_pretrained(
    finetuning_model,
    quantization_config=quantization_config,
    device_map="auto",
)
quantized_tokenizer = AutoTokenizer.from_pretrained(finetuning_model)

quantized_pipe = pipeline("text-generation", model=quantized_model, tokenizer=quantized_tokenizer)

question = "실직으로 소득이 없어졌는데, 지역가입자 보험료를 감면받거나 납부를 유예할 수 있는 제도가 있나요? 있다면 신청 조건이 어떻게 되나요?"
prompt = f"### 질문:{question}\n\n### 답변:"

print("\n" + "="*60)
print(f"질문: {prompt.split('### 답변:')[0].replace('### 질문: ', '').strip()}")
print("="*60 + "\n")

# 파인튜닝 & 양자화 모델 추론
quantized_result = quantized_pipe(
    prompt,
    max_new_tokens=150,
    temperature=0.7,
    eos_token_id=tokenizer.eos_token_id,
    return_full_text=False
)
print(quantized_result[0]['generated_text'])