Views
No views yet
atsujita/llm-jp-3-13b-finetune)を用いて入力データ(elyza-tasks-100-TV_0.jsonl)を推論し、その結果を{adapter_id}-outputs.jsonlというファイルに出力できます。1!pip install -U bitsandbytes
2!pip install -U transformers
3!pip install -U accelerate
4!pip install -U datasets
5!pip install -U pef
6!pip install ipywidgets --upgrade1import torch
2from transformers import (
3 AutoModelForCausalLM,
4 AutoTokenizer,
5 BitsAndBytesConfig,
6)
7from peft import PeftModel
8import json
9from tqdm import tqdm
10import re
11
12model_id = "llm-jp/llm-jp-3-13b"
13adapter_id = "atsujita/llm-jp-3-13b-finetune"
14
15# QLoRA用の設定
16bnb_config = BitsAndBytesConfig(
17 load_in_4bit=True,
18 bnb_4bit_quant_type="nf4",
19 bnb_4bit_compute_dtype=torch.bfloat16,
20)
21
22# モデル読み込み
23model = AutoModelForCausalLM.from_pretrained(
24 model_id,
25 quantization_config=bnb_config,
26 device_map="auto",
27 token=HF_TOKEN
28)
29
30tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, token=HF_TOKEN)
31
32# Peftモデルを適用
33model = PeftModel.from_pretrained(model, adapter_id, token=HF_TOKEN)./elyza-tasks-100-TV_0.jsonlというファイルからデータセットをロードします。1datasets = []
2with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
3 item = ""
4 for line in f:
5 line = line.strip()
6 item += line
7 if item.endswith("}"):
8 datasets.append(json.loads(item))
9 item = ""1results = []
2for data in tqdm(datasets):
3 input_data = data["input"]
4
5 prompt = f"""### 指示
6{input_data}
7### 回答
8"""
9
10 tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
11 attention_mask = torch.ones_like(tokenized_input)
12 with torch.no_grad():
13 outputs = model.generate(
14 tokenized_input,
15 attention_mask=attention_mask,
16 max_new_tokens=200,
17 do_sample=False,
18 repetition_penalty=1.2,
19 pad_token_id=tokenizer.eos_token_id
20 )[0]
21
22 output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
23
24 # 結果を保存
25 results.append({
26 "input": input_data,
27 "output": output
28 })1jsonl_id = re.sub(".*/", "", adapter_id)
2with open(f"./{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
3 for result in results:
4 json.dump(result, f, ensure_ascii=False)
5 f.write('\n')