Views
No views yet
1!pip install -U transformers
2!pip install -U bitsandbytes
3!pip install -U accelerate
4!pip install -U datasets
5!pip install -U peft
6!pip install -U trl==0.12.01from transformers import (
2 AutoModelForCausalLM,
3 AutoTokenizer,
4 BitsAndBytesConfig,
5)
6from peft import PeftModel
7import torch
8
9HF_TOKEN = "AVAILABLE YOUR-HF-TOKEN"
10
11model_name = "llm-jp/llm-jp-3-13b"
12adapter_name = "r-yuba62/llm-jp-3-13b-finetune"
13
14bnb_config = BitsAndBytesConfig(
15 load_in_4bit=True,
16 bnb_4bit_quant_type="nf4",
17 bnb_4bit_compute_dtype=torch.bfloat16,
18)
19
20# モデルとトークナイザーをロード
21model = AutoModelForCausalLM.from_pretrained(
22 model_name,
23 quantization_config=bnb_config,
24 device_map="auto",
25 token=HF_TOKEN
26)
27
28tokenizer = AutoTokenizer.from_pretrained(
29 model_name,
30 trust_remote_code=True,
31 token=HF_TOKEN
32)
33
34# PEFTアダプターを適用
35model = PeftModel.from_pretrained(model, adapter_name, token=HF_TOKEN)
36
37def get_response(input_text):
38 prompt = f"""### 指示
39 {input_text}
40 ### 回答：
41 """
42 # トークナイズ処理
43 tokenized_input = tokenizer(
44 prompt,
45 return_tensors="pt",
46 padding=True,
47 truncation=True,
48 max_length=512
49 )
50
51 input_ids = tokenized_input["input_ids"].to(model.device)
52 attention_mask = tokenized_input["attention_mask"].to(model.device)
53
54 # モデル生成
55 with torch.no_grad():
56 outputs = model.generate(
57 input_ids=input_ids,
58 attention_mask=attention_mask,
59 max_new_tokens=512,
60 do_sample=False,
61 repetition_penalty=1.2,
62 pad_token_id=tokenizer.eos_token_id
63 )
64
65 # 出力のデコード
66 output_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
67 return output_text
68
69input_text = "xxxを教えてください"
70response = get_response(input_text)
71print(response)| Language | Dataset | description |
|---|---|---|
| Japanese | ichikara-instruction-003-001-1.json | A manually constructed instruction dataset |