Views
No views yet
1# 必要なライブラリを読み込み
2from unsloth import FastLanguageModel
3from peft import PeftModel
4import torch
5import json
6from tqdm import tqdm
7import re1# Hugging Faceで取得したToken
2HF_TOKEN = "{Your hugging face token}"
3
4# モデルのIDと、LoRAのアダプタ名
5model_id = "llm-jp/llm-jp-3-13b"
6adapter_id = "nishimura999/llm-jp-3-13b-it-v108_lora"1# unslothのFastLanguageModelで元のモデルをロード。
2dtype = None
3load_in_4bit = True
4model, tokenizer = FastLanguageModel.from_pretrained(
5 model_name=model_id,
6 dtype=dtype,
7 load_in_4bit=load_in_4bit,
8 trust_remote_code=True,
9)
10# 元のモデルにLoRAのアダプタを統合。
11model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)1# データセットの読み込み。
2datasets = []
3with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
4 item = ""
5 for line in f:
6 line = line.strip()
7 item += line
8 if item.endswith("}"):
9 datasets.append(json.loads(item))
10 item = ""1# モデルを用いてタスクの推論。
2
3FastLanguageModel.for_inference(model)
4
5results = []
6for dt in tqdm(datasets):
7 input = dt["input"]
8
9 prompt = f"""### 指示\n{input}\n### 回答\n"""
10
11 inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
12
13 outputs = model.generate(**inputs, max_new_tokens = 1024, use_cache = True, do_sample=False, repetition_penalty=1.2)
14 prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).split('\n### 回答')[-1]
15
16 results.append({"task_id": dt["task_id"], "input": input, "output": prediction})1# 結果をjsonlで保存。
2json_file_id = re.sub(".*/", "", adapter_id)
3with open(f"./{json_file_id}_output.jsonl", 'w', encoding='utf-8') as f:
4 for result in results:
5 json.dump(result, f, ensure_ascii=False)
6 f.write('\n')