Views
No views yet
1pip install unsloth
2pip install --no-deps --upgrade "flash-attn>=2.6.3"
3pip install -U ragatouille
4pip install fugashi unidic-lite1from datasets import concatenate_datasets, load_dataset
2from unsloth import FastLanguageModel
3import random
4import json
5
6from huggingface_hub import login
7from google.colab import userdata
8login(userdata.get('HFtoken'))
9
10
11with open("elyza-tasks-100-TV_0.jsonl","r",encoding='utf-8') as f:
12 tasks = [json.loads(l) for l in f.readlines()]
13
14model_name = "tomo1222/Gemma2-27b-ft-jp-r64_alpha64"
15
16
17max_seq_length = 4096
18
19model, tokenizer = FastLanguageModel.from_pretrained(
20 model_name = model_name,
21 max_seq_length = max_seq_length,
22 dtype = None,
23 load_in_4bit = True,
24)
25
26# google/gemma-2-9bのテンプレート
27tokenizer.chat_template = """
28{{ bos_token }}{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '\n' + message['content'] | trim + '<end_of_turn>\n' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model\n'}}{% endif %}
29"""
30FastLanguageModel.for_inference(model) # Enable native 2x faster inference
31
32dataset = load_dataset("tomo1222/Japanese-QA111dataset")
33ref_tasks = list(dataset["train"])
34ref_tasks_input = [task["input"] for task in ref_tasks]
35
36dic = {}
37dic_input = {}
38for i, task in enumerate(ref_tasks):
39 dic[ref_tasks_input[i]] = task["output"]
40 dic_input[ref_tasks_input[i]] = task["input"]
41
42"""# 2. RAGのロード"""
43
44from ragatouille import RAGPretrainedModel
45RAG = RAGPretrainedModel.from_pretrained("bclavie/JaColBERTv2")
46RAG.encode(ref_tasks_input)
47
48def search_ref_input(input, k=10):
49 retreived=RAG.search_encoded_docs(query=input,k=k)
50 print(retreived)
51 text ="質問・文章をよく読んで、正確で親切な回答を書きなさい。\n"
52 for data in retreived[::-1]: # inverse order
53 key = data["content"]
54 output = dic[key]
55 input = dic_input[key]
56 text+="### 質問:\n"+input+"\n\n### 回答:\n"+output+"\n\n\n"
57 return text
58
59"""# Prompt"""
60output_data=[]
61
62for i, task in enumerate(tasks):
63 text = (
64 search_ref_input(task["input"], 20)
65 + "あなたは日本語が堪能な優秀な人間です。\n"
66 + "**文脈**を踏まえて、改行と箇条書きを駆使して、日本語で**詳細に**書きなさい。\n"
67 + "優秀な人間になりきって、推測をいれずに根拠をもってわかりやすく答えてください。"
68 + f"### 質問:\n{task['input']}\n\n### 回答:\n"
69 )
70 print(task["input"])
71 inputs = tokenizer(text, return_tensors="pt").to("cuda")
72 print(len(inputs['input_ids'][0]))
73 output = model.generate(**inputs, max_new_tokens=1024,repetition_penalty=1.1,use_cache=True,
74 bad_words_ids = [tokenizer.encode("質問", add_special_tokens=False),
75 tokenizer.encode("###", add_special_tokens=False),
76 tokenizer.encode("#", add_special_tokens=False),
77 tokenizer.encode("##", add_special_tokens=False),
78 tokenizer.encode("---", add_special_tokens=False),
79 tokenizer.encode("<h3>", add_special_tokens=False),
80 tokenizer.encode("filepath", add_special_tokens=False),
81 tokenizer.encode("> ", add_special_tokens=False),
82 ]
83 )
84
85 output_text = tokenizer.decode(output[0][inputs.input_ids.size(1):], skip_special_tokens=True).strip()
86 print(i,output_text)
87 print("---")
88 output_data.append({"task_id":i,"output":output_text})
89
90with open("output.jsonl","w",encoding="utf-8") as f:
91 for result in output_data:
92 json.dump(result, f, ensure_ascii=False)
93 f.write('\n')