Views
No views yet
1
2 #!pip install transformers==4.40.0 accelerate
3 import os
4 import torch
5 from transformers import AutoTokenizer, AutoModelForCausalLM
6
7 model_id = 'Dongwookss/small_fut_final'
8
9 tokenizer = AutoTokenizer.from_pretrained(model_id)
10 model = AutoModelForCausalLM.from_pretrained(
11 model_id,
12 torch_dtype=torch.bfloat16,
13 device_map="auto",
14 )
15 model.eval()1from transformers import TextStreamer
2PROMPT = '''Below is an instruction that describes a task. Write a response that appropriately completes the request.
3제시하는 context에서만 대답하고 context에 없는 내용은 모르겠다고 대답해'''
4
5messages = [
6 {"role": "system", "content": f"{PROMPT}"},
7 {"role": "user", "content": f"{instruction}"}
8 ]
9
10input_ids = tokenizer.apply_chat_template(
11 messages,
12 add_generation_prompt=True,
13 return_tensors="pt"
14).to(model.device)
15
16terminators = [
17 tokenizer.eos_token_id,
18 tokenizer.convert_tokens_to_ids("<|eot_id|>")
19]
20
21text_streamer = TextStreamer(tokenizer)
22_ = model.generate(
23 input_ids,
24 max_new_tokens=4096,
25 eos_token_id=terminators,
26 do_sample=True,
27 streamer = text_streamer,
28 temperature=0.6,
29 top_p=0.9,
30 repetition_penalty = 1.1
31)
32 r=32,
lora_alpha=64, # QLoRA : alpha = r/2 // LoRA : alpha =r*2
lora_dropout=0.05,
target_modules=[
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
], # 타겟 모듈