Views
No views yet
dakesan0-inference-testcode.ipynb1from unsloth import FastLanguageModel
2from peft import PeftModel
3import torch
4import json
5from tqdm import tqdm
6import re
7import datasets
8
9model_id = "llm-jp/llm-jp-3-13b"
10adapter_id = "poprap/llm-jp-3-13b-it-2-3"
11adapter_dpo_id = "poprap/llm-jp-3-13b-dpo"
12
13dtype = None
14load_in_4bit = True
15
16model, tokenizer = FastLanguageModel.from_pretrained(
17 model_name=model_id,
18 dtype=dtype,
19 load_in_4bit=load_in_4bit,
20 trust_remote_code=True,
21)
22
23model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)
24model = PeftModel.from_pretrained(model, adapter_dpo_id, token = HF_TOKEN)
25
26ds = []
27
28with open("elyza-tasks-100-TV_0.jsonl", "r") as f:
29 item = ""
30 for line in f:
31 line = line.strip()
32 item += line
33 if item.endswith("}"):
34 ds.append(json.loads(item))
35 item = ""
36
37# 推論するためにモデルのモードを変更
38FastLanguageModel.for_inference(model)
39
40results = []
41for dt in tqdm(ds):
42 input = dt["input"]
43
44 prompt = f"""### 指示\n{input}\n上記指示に簡潔に回答してください。\n### 回答\n"""
45
46 inputs = tokenizer([prompt], return_tensors = "pt").to(model.device)
47
48 outputs = model.generate(
49 **inputs,
50 max_new_tokens=1024,
51 use_cache = True,
52 do_sample=False,
53 repetition_penalty=1.2
54 )
55 prediction = tokenizer.decode(outputs[0], skip_special_tokens=True).split('\n### 回答')[-1]
56
57 results.append({"task_id": dt['task_id'], "input": input, "output": prediction})
58
59json_file_id = re.sub(".*/", "", adapter_id)
60with open(f"{json_file_id}_output.jsonl", 'w', encoding='utf-8') as f:
61 for result in results:
62 json.dump(result, f, ensure_ascii=False)
63 f.write('\n')