Views
No views yet
1!pip install -U bitsandbytes
2!pip install -U transformers
3!pip install -U accelerate
4!pip install -U datasets
5!pip install -U peft
6!pip install ipywidgets --upgrade
7
8from transformers import (
9 AutoModelForCausalLM,
10 AutoTokenizer,
11 BitsAndBytesConfig,
12 TrainingArguments,
13 logging,
14)
15from peft import (
16 LoraConfig,
17 PeftModel,
18 get_peft_model,
19)
20import os, torch, gc
21from datasets import load_dataset
22import bitsandbytes as bnb
23from trl import SFTTrainer
24
25# Hugging Face Token
26HF_TOKEN = "mytoken"
27
28model_id = "models/models--llm-jp--llm-jp-3-13b/snapshots/cd3823f4c1fcbb0ad2e2af46036ab1b0ca13192a"
29adapter_id = "kozamoto/llm-jp-3-13b-finetune05" # こちらにアップロードしたHugging FaceのIDを指定してください。
30
31# QLoRA config
32bnb_config = BitsAndBytesConfig(
33 load_in_4bit=True,
34 bnb_4bit_quant_type="nf4",
35 bnb_4bit_compute_dtype=torch.bfloat16,
36)
37
38# Load model
39model = AutoModelForCausalLM.from_pretrained(
40 model_id,
41 quantization_config=bnb_config,
42 device_map="auto",
43 token = HF_TOKEN
44)
45
46# Load tokenizer
47tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True, token = HF_TOKEN)
48
49# 元のモデルにLoRAのアダプタを統合。
50model = PeftModel.from_pretrained(model, adapter_id, token = HF_TOKEN)
51
52import json
53datasets = []
54with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
55 item = ""
56 for line in f:
57 line = line.strip()
58 item += line
59 if item.endswith("}"):
60 datasets.append(json.loads(item))
61 item = ""
62
63from tqdm import tqdm
64
65results = []
66for data in tqdm(datasets):
67
68 input = data["input"]
69
70 prompt = f"""### 指示
71 {input}
72 ### 回答
73 """
74
75 tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
76 attention_mask = torch.ones_like(tokenized_input)
77
78 with torch.no_grad():
79 outputs = model.generate(
80 tokenized_input,
81 attention_mask=attention_mask,
82 max_new_tokens=100,
83 do_sample=False,
84 repetition_penalty=1.2,
85 pad_token_id=tokenizer.eos_token_id
86 )[0]
87 output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
88
89 results.append({"task_id": data["task_id"], "input": input, "output": output})
90
91import re
92jsonl_id = re.sub(".*/", "", new_model_id)
93with open(f"./{jsonl_id}-outputs.jsonl", 'w', encoding='utf-8') as f:
94 for result in results:
95 json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
96 f.write('\n')1training_arguments = TrainingArguments(
2 output_dir=new_model_id,
3 per_device_train_batch_size=1,
4 gradient_accumulation_steps=2,
5 optim="paged_adamw_32bit",
6 num_train_epochs=3,
7 logging_strategy="steps",
8 logging_steps=50,
9 warmup_steps=500,
10 save_steps=500,
11 save_total_limit = 3,
12 max_steps = -1,
13 learning_rate=3e-5,
14 fp16=True,
15 bf16=False,
16 seed = 3407,
17 group_by_length=True,
18 report_to="none"
19)