Views
No views yet
1#!pip install -U pip
2#!pip install -U bitsandbytes
3#!pip install -U transformers
4#!pip install -U accelerate
5#!pip install -U datasets
6#!pip install -U peft
7
8from transformers import (
9 AutoModelForCausalLM,
10 AutoTokenizer,
11 BitsAndBytesConfig,
12)
13from peft import PeftModel
14import torch
15from tqdm import tqdm
16import json
17
18# Hugging Faceで取得したTokenをこちらに貼る。
19HF_TOKEN=""
20
21# model
22model_id = "llm-jp/llm-jp-3-13b"
23adapter_id = "shigedon/llm-jp-3-13b-finetune-14-Dec-20.46.10"
24
25# QLoRA config
26bnb_config = BitsAndBytesConfig(
27 load_in_4bit=True,
28 bnb_4bit_quant_type="nf4",
29 bnb_4bit_compute_dtype=torch.bfloat16,
30)
31
32# Load model
33model = AutoModelForCausalLM.from_pretrained(
34 model_id,
35 quantization_config=bnb_config,
36 device_map="auto",
37 token = HF_TOKEN
38)
39
40# Load tokenizer
41tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, token = HF_TOKEN)
42
43# 元のモデルにLoRAのアダプタを統合。
44model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)
45
46# dataset
47datasets = []
48with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
49 item = ""
50 for line in f:
51 line = line.strip()
52 item += line
53 if item.endswith("}"):
54 datasets.append(json.loads(item))
55 item = ""
56
57results = []
58for data in tqdm(datasets):
59
60 input = data["input"]
61 prompt = f"""### 指示
62 {input}
63 ### 回答
64 """
65
66 input_ids = tokenizer(prompt, return_tensors="pt",return_token_type_ids=False,).to(model.device)
67 outputs = model.generate(**input_ids, max_new_tokens=512, do_sample=False, repetition_penalty=1.2,)
68 output = tokenizer.decode(outputs[0][input_ids.input_ids.size(1):], skip_special_tokens=True)
69
70 results.append({"task_id": data["task_id"], "input": input, "output": output})
71
72
73# llmjp
74results = []
75for data in tqdm(datasets):
76
77 input = data["input"]
78
79 prompt = f"""### 指示
80 {input}
81 ### 回答
82 """
83
84 tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
85 attention_mask = torch.ones_like(tokenized_input)
86 with torch.no_grad():
87 outputs = model.generate(
88 tokenized_input,
89 attention_mask=attention_mask,
90 max_new_tokens=100,
91 do_sample=False,
92 repetition_penalty=1.2,
93 pad_token_id=tokenizer.eos_token_id
94 )[0]
95 output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
96
97 results.append({"task_id": data["task_id"], "input": input, "output": output})
98
99# jsonl
100import re
101jsonl_id = re.sub(".*/", "", adapter_id)
102with open(f"./{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
103 for result in results:
104 json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
105 f.write('\n')
106