RichardErkhov/

jojo0217_-_ChatSKKU5.8B-4bits

from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer

model_name="jojo0217/ChatSKKU5.8B"
model = AutoModelForCausalLM.from_pretrained(
            model_name,
            device_map="auto",
            load_in_8bit=True,#만약 양자화 끄고 싶다면 false
        )
tokenizer = AutoTokenizer.from_pretrained(model_name)
pipe = pipeline(
            "text-generation",
            model=model,
            tokenizer=model_name,
            device_map="auto"
        )

def answer(message):
  prompt=f"아래는 작업을 설명하는 명령어입니다. 요청을 적절히 완료하는 응답을 작성하세요.\n\n### 명령어:\n{message}"
  ans = pipe(
        prompt + "\n\n### 응답:",
        do_sample=True,
        max_new_tokens=512,
        temperature=0.7,
        repetition_penalty = 1.0,
        return_full_text=False,
        eos_token_id=2,
    )
  msg = ans[0]["generated_text"]
  return msg
answer('성균관대학교에대해 알려줘')