Views
No views yet
1!pip install unsloth
2!pip uninstall unsloth -y && pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
3!pip install -U torch
4!pip install -U peft1from unsloth import FastLanguageModel
2from peft import PeftModel
3import torch
4import json
5from tqdm import tqdm
6import re1""" ベースとなるモデルと学習したLoRAのアダプタ """
2model_id = "llm-jp/llm-jp-3-13b" # ベースモデル
3adapter_id = "hiroshi1991/llm-jp-3-13b-it" # アダプタ
4HF_TOKEN= "your_token" # あなたのtokenを入力：read権限でも可
5
6""" UnslothのFastLanguageModelでベースモデルをロード """
7dtype = None
8load_in_4bit = True
9
10model, tokenizer = FastLanguageModel.from_pretrained(
11 model_name=model_id,
12 dtype=dtype,
13 load_in_4bit=load_in_4bit,
14 trust_remote_code=True,
15 token=HF_TOKEN
16)
17
18""" ベースモデルにLoRAアダプタを統合 """
19model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)1""" 今回のタスクであるelyza-tasks-100-TV_0.jsonlの読込み """
2datasets = []
3with open("./elyza-tasks-100-TV_0.jsonl", "r") as f: # fileの場所に応じてpathを書き替えてください
4 item = ""
5 for line in f:
6 line = line.strip()
7 item += line
8 if item.endswith("}"):
9 datasets.append(json.loads(item))
10 item = ""
11
12""" モデルのモードを推論用に変更 """
13FastLanguageModel.for_inference(model)
14
15results = []
16for dt in tqdm(datasets):
17 input = dt["input"]
18
19 prompt = f"""### 指示\n{input}\n### 回答\n"""
20
21 inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
22
23 outputs = model.generate(**inputs, max_new_tokens = 512, use_cache = True, do_sample=False, repetition_penalty=1.2)
24 prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).split('\n### 回答')[-1]
25 prediction = re.sub(r"[*#]", "", prediction)
26
27 results.append({"task_id": dt["task_id"], "input": input, "output": prediction})
28
29""" 推論した結果をjsonlで保存 """
30model_name = re.sub(".*/", "", adapter_id)
31with open(f"./{model_name}-my-original-outputs.jsonl", 'w', encoding='utf-8') as f: # 保存したい場所・ファイル名に適宜変更してください
32 for result in results:
33 json.dump(result, f, ensure_ascii=False)
34 f.write('\n')