Views
No views yet
1from transformers import AutoModelForCausalLM, AutoTokenizer
2from peft import PeftModel
3import torch
4
5base_model = AutoModelForCausalLM.from_pretrained(
6 "togethercomputer/GPT-NeoXT-Chat-Base-20B",
7 load_in_4bit=True,
8 device_map="auto",
9 trust_remote_code=True
10)
11
12model = PeftModel.from_pretrained(base_model, "takumi123xxx/gpt-neox-20b-qlora-japanese")
13
14tokenizer = AutoTokenizer.from_pretrained("togethercomputer/GPT-NeoXT-Chat-Base-20B")
15
16prompt = "### Instruction:\n日本の首都はどこですか？\n\n### Response:\n"
17inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
18
19with torch.no_grad():
20 outputs = model.generate(
21 **inputs,
22 max_new_tokens=100,
23 temperature=0.7,
24 do_sample=True,
25 top_p=0.9
26 )
27
28response = tokenizer.decode(outputs[0], skip_special_tokens=True)
29print(response)