Views
No views yet
%%capture
!pip install unsloth
!pip uninstall unsloth -y && pip install --upgrade --no-cache-dir "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
!pip install -U torch
!pip install -U peft1from unsloth import FastLanguageModel
2from peft import PeftModel
3import torch
4import json
5from tqdm import tqdm
6import re1model_id = "llm-jp/llm-jp-3-13b"
2adapter_id = "YuJaq/llm-jp-3-13b-it_YuT-LoRA2"
3
4HF_TOKEN = "HF_TOKEN"
5
6dtype = None
7load_in_4bit = True
8
9# unslothのFastLanguageModelで元のモデルをロード
10model, tokenizer = FastLanguageModel.from_pretrained(
11 model_name=model_id,
12 dtype=dtype,
13 load_in_4bit = load_in_4bit,
14 trust_remote_code = True,
15)
16
17# 元のモデルにLoRAのアダプタを統合
18model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)
19
20# タスクとなるデータの読み込み。
21# 事前にデータをアップロードしてください。
22datasets = []
23with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
24 item = ""
25 for line in f:
26 line = line.strip()
27 item += line
28 if item.endswith("}"):
29 datasets.append(json.loads(item))
30 item = ""
31
32# タスクの推論
33# モデルのモードを変更
34FastLanguageModel.for_inference(model)
35
36results = []
37for dt in tqdm(datasets):
38 input = dt["input"]
39
40 prompt = f"""### 指示\n{input}\n### 回答\n"""
41
42 inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
43
44 outputs = model.generate(**inputs, max_new_tokens = 512, use_cache = True, do_sample = False, repetition_penalty = 1.2)
45 prediction = tokenizer.decode(outputs[0], skip_special_tokens = True).split('\n### 回答')[-1]
46
47 results.append({"task_id": dt["task_id"], "input": input, "output": prediction})
48
49
50# 結果をjsonlで保存
51json_file_id = re.sub(".*/", "", adapter_id)
52with open(f"/content/{json_file_id}_output.jsonl", 'w', encoding = 'utf-8') as f:
53 for result in results:
54 json.dump(result, f, ensure_ascii = False)
55 f.write('\n')