Views
No views yet
llm-jp/llm-jp-3-13b + tmdoi/llm-jp-3-13b-ft-CHGPara-myds-20241216c-ep1)を用いて入力データ(elyza-tasks-100-TV_0.jsonl)を推論し、その結果を{adapter_id}-outputs.jsonlというファイルに出力できます。1!pip install -U bitsandbytes
2!pip install -U transformers
3!pip install -U accelerate
4!pip install -U datasets
5!pip install -U peft
6!pip install ipywidgets --upgrade
7
8from transformers import (
9 AutoModelForCausalLM,
10 AutoTokenizer,
11 BitsAndBytesConfig,
12)
13from peft import PeftModel
14import torch
15from tqdm import tqdm
16import json
17
18# Hugging Faceで取得したToken
19HF_TOKEN = "Hugging Face Token"
20
21# ベースとなるモデルと学習したLoRAのアダプタ
22model_id = "llm-jp/llm-jp-3-13b"
23adapter_id = "tmdoi/llm-jp-3-13b-ft-CHGPara-myds-20241215f-ep1"
24
25# QLoRA config
26bnb_config = BitsAndBytesConfig(
27 load_in_4bit=True,
28 bnb_4bit_quant_type="nf4",
29 bnb_4bit_compute_dtype=torch.bfloat16,
30)
31
32# Load model
33model = AutoModelForCausalLM.from_pretrained(
34 model_id,
35 quantization_config=bnb_config,
36 device_map="auto",
37 token = HF_TOKEN
38)
39
40# Load tokenizer
41tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, token = HF_TOKEN)
42
43# 元のモデルにLoRAのアダプタを統合
44model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)
45
46# データセットの読み込み。
47datasets = []
48
49# with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
50with open("~/elyza-tasks-100-TV_0.jsonl", "r") as f:
51 item = ""
52 for line in f:
53 line = line.strip()
54 item += line
55 if item.endswith("}"):
56 datasets.append(json.loads(item))
57 item = ""
58
59# データセットの読み込み。
60# omnicampusの開発環境では、左にタスクのjsonlをドラッグアンドドロップしてから実行。
61datasets = []
62
63# with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
64with open("/content/drive/MyDrive/SFT_outputs/elyza-tasks-100-TV_0.jsonl", "r") as f:
65 item = ""
66 for line in f:
67 line = line.strip()
68 item += line
69 if item.endswith("}"):
70 datasets.append(json.loads(item))
71 item = ""
72
73# 推論実行
74# llmjp
75print(adapter_id)
76results = []
77for data in tqdm(datasets):
78
79 input = data["input"]
80
81 prompt = f"""### 指示
82 {input}
83 ### 回答
84 """
85
86 tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
87 attention_mask = torch.ones_like(tokenized_input)
88 with torch.no_grad():
89 outputs = model.generate(
90 tokenized_input,
91 attention_mask=attention_mask,
92 max_new_tokens=512,
93 do_sample=False,
94 repetition_penalty=1.2,
95 pad_token_id=tokenizer.eos_token_id
96 )[0]
97 output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
98
99 #目視確認用の出力
100 print("==INPUT===")
101 print(input)
102 print("==OUTPUT===")
103 print(output)
104
105 results.append({"task_id": data["task_id"], "input": input, "output": output})
106
107
108## 出力の保存
109import re
110jsonl_id = re.sub(".*/", "", adapter_id)
111with open(f"{output_GD_dir}/{new_model_id}/{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
112 for result in results:
113 json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
114 f.write('\n')
115