Views
No views yet
pip install -U ipywidgets
pip install transformers==4.46.3
pip install -U bitsandbytes
pip install -U accelerate
pip install -U datasets
pip install -U peft==0.13.21### データセットの読み込み ###
2datasets = []
3with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
4 item = ""
5 for line in f:
6 line = line.strip()
7 item += line
8 if item.endswith("}"):
9 datasets.append(json.loads(item))
10 item = ""1from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
2from peft import PeftModel
3import torch
4from tqdm import tqdm
5import json
6import re
7
8# Hugging Face Access Tokenキー設定
9HF_TOKEN = "有効なHuggingFaceトークン"
10
11
12# model ids
13model_id = "models/models--llm-jp--llm-jp-3-13b/snapshots/cd3823f4c1fcbb0ad2e2af46036ab1b0ca13192a"
14adapter_id = "tnk-2727/llm-jp-3-13b-finetune"
15adapter_dpo_id = adapter_id
16
17
18### モデル設定 ###
19# QLoRA用の設定
20bnb_config = BitsAndBytesConfig(
21 load_in_4bit=True,
22 bnb_4bit_quant_type="nf4",
23 bnb_4bit_compute_dtype=torch.bfloat16,
24)
25
26# モデル読み込み
27model = AutoModelForCausalLM.from_pretrained(
28 model_id,
29 quantization_config=bnb_config,
30 device_map="auto",
31 token=HF_TOKEN
32)
33tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, token=HF_TOKEN)
34
35# 元のモデルにLoRAのアダプタを統合
36model = PeftModel.from_pretrained(model, adapter_id, token=HF_TOKEN)
37# LoRAのモデルにDPOのアダプタを統合
38model = PeftModel.from_pretrained(model, adapter_dpo_id, token = HF_TOKEN)
39
40
41### 推論 ###
42results = []
43for data in tqdm(datasets):
44
45 input = data["input"]
46
47 prompt = f"""### 指示
48 {input}
49 ### 回答
50 """
51
52 tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
53 attention_mask = torch.ones_like(tokenized_input)
54 with torch.no_grad():
55 outputs = model.generate(
56 tokenized_input,
57 attention_mask=attention_mask,
58 max_new_tokens=100,
59 do_sample=False,
60 repetition_penalty=1.2,
61 pad_token_id=tokenizer.eos_token_id
62 )[0]
63 output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
64
65 results.append({"task_id": data["task_id"], "input": input, "output": output})
66
67### ファイル出力 ###
68# 出力結果をファイルに出力
69jsonl_id = re.sub(".*/", "", adapter_dpo_id)
70with open(f"./{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
71 for result in results:
72 json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
73 f.write('\n')