Views
No views yet
elyza-tasks-100-TV_0.jsonl)を推論し、その結果を{adapter_id}-outputs.jsonlというファイルに出力できます。1!pip install -U bitsandbytes
2!pip install -U transformers
3!pip install -U accelerate
4!pip install -U datasets
5!pip install -U pef
6!pip install ipywidgets --upgrade1from google.colab import userdata
2HF_TOKEN = userdata.get('HF_TOKEN')1import torch
2from transformers import (
3 AutoModelForCausalLM,
4 AutoTokenizer,
5 BitsAndBytesConfig,
6)
7from peft import PeftModel
8import json
9from tqdm import tqdm
10import re
11model_id = "llm-jp/llm-jp-3-13b"
12adapter_id = "tajimataso/llm-jp-3-13b-finetune_1216"
13
14# QLoRA用の設定
15bnb_config = BitsAndBytesConfig(
16 load_in_4bit=True,
17 bnb_4bit_quant_type="nf4",
18 bnb_4bit_compute_dtype=torch.bfloat16,
19)
20# モデル読み込み
21model = AutoModelForCausalLM.from_pretrained(
22 model_id,
23 quantization_config=bnb_config,
24 device_map="auto",
25 token=HF_TOKEN
26)
27tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, token=HF_TOKEN)
28# Peftモデルを適用
29model = PeftModel.from_pretrained(model, adapter_id, token=HF_TOKEN)./elyza-tasks-100-TV_0.jsonlというファイルからデータセットをロードします。1datasets = []
2with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
3 item = ""
4 for line in f:
5 line = line.strip()
6 item += line
7 if item.endswith("}"):
8 datasets.append(json.loads(item))
9 item = ""1results = []
2for data in tqdm(datasets):
3 input_data = data["input"]
4
5 prompt = f"""### 指示
6{input_data}
7### 回答
8"""
9 tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
10 attention_mask = torch.ones_like(tokenized_input)
11 with torch.no_grad():
12 outputs = model.generate(
13 tokenized_input,
14 attention_mask=attention_mask,
15 max_new_tokens=200,
16 do_sample=False,
17 repetition_penalty=1.2,
18 pad_token_id=tokenizer.eos_token_id
19 )[0]
20 output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
21
22 # 結果を保存
23 results.append({
24 "input": input_data,
25 "output": output
26 })1jsonl_id = re.sub(".*/", "", adapter_id)
2with open(f"./{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
3 for result in results:
4 json.dump(result, f, ensure_ascii=False)
5 f.write('\n')