Views
No views yet
1!pip install -U bitsandbytes
2!pip install -U transformers
3!pip install -U accelerate
4!pip install -U datasets
5!pip install -U peft1from transformers import (
2 AutoModelForCausalLM,
3 AutoTokenizer,
4 BitsAndBytesConfig,
5)
6from peft import PeftModel
7import torch
8from tqdm import tqdm
9import json
10import re
11
12# Hugging Face Token (recommended to set via environment variable)
13HF_TOKEN = "YOUR_HF_ACCESS_TOKEN"
14
15# Model and adapter IDs
16# base_model_id = "models/models--llm-jp--llm-jp-3-13b/snapshots/cd3823f4c1fcbb0ad2e2af46036ab1b0ca13192a"
17base_model_id = "llm-jp/llm-jp-3-13b" # Base model
18adapter_id = "sasakipeter/llm-jp-3-13b-finetune"
19
20# QLoRA (4-bit quantization) configuration
21bnb_config = BitsAndBytesConfig(
22 load_in_4bit=True,
23 bnb_4bit_quant_type="nf4",
24 bnb_4bit_compute_dtype=torch.bfloat16,
25)1# Load base model with 4-bit quantization
2model = AutoModelForCausalLM.from_pretrained(
3 base_model_id,
4 quantization_config=bnb_config,
5 device_map="auto",
6 token=HF_TOKEN
7)
8
9# Load tokenizer
10tokenizer = AutoTokenizer.from_pretrained(
11 base_model_id,
12 trust_remote_code=True,
13 token=HF_TOKEN
14)
15
16# Integrate LoRA adapter into the base model
17model = PeftModel.from_pretrained(model, adapter_id, token=HF_TOKEN)
18model.config.use_cache = False[elyza-tasks-100](https://huggingface.co/datasets/elyza/ELYZA-tasks-100)1# loading dataset
2datasets = []
3with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
4 item = ""
5 for line in f:
6 line = line.strip()
7 item += line
8 if item.endswith("}"):
9 datasets.append(json.loads(item))
10 item = ""
11
12# execute inference
13results = []
14for data in tqdm(datasets):
15
16 input_text = data["input"]
17
18 prompt = f"""### 指示
19 {input_text}
20 ### 回答
21 """
22
23 tokenized_input = tokenizer.encode(prompt, add_special_tokens=False, return_tensors="pt").to(model.device)
24 attention_mask = torch.ones_like(tokenized_input)
25
26 with torch.no_grad():
27 outputs = model.generate(
28 tokenized_input,
29 attention_mask=attention_mask,
30 max_new_tokens=100,
31 do_sample=False,
32 repetition_penalty=1.2,
33 pad_token_id=tokenizer.eos_token_id
34 )[0]
35 output = tokenizer.decode(outputs[tokenized_input.size(1):], skip_special_tokens=True)
36
37 results.append({"task_id": data["task_id"], "input": input_text, "output": output})
38
39jsonl_id = re.sub(".*/", "", adapter_id)
40with open(f"./{jsonl_id}-outputs-validation.jsonl", 'w', encoding='utf-8') as f:
41 for result in results:
42 json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
43 f.write('\n')