Views
No views yet
| Domain | Accuracy |
|---|---|
| Professional Psychology | 76% |
| Management | 74% |
| Sociology | 75% |
1from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
2from peft import PeftModel
3import torch
4
5# Define the model IDs
6base_model_name_or_path = "alibidaran/Platio_merged_model" # The base Llama-3-8B-Instruct model
7
8# 1. Configure 4-bit quantization
9bnb_config = BitsAndBytesConfig(
10 load_in_4bit=True,
11 bnb_4bit_use_double_quant=True,
12 bnb_4bit_quant_type="nf4",
13 bnb_4bit_compute_dtype=torch.float16
14)
15
16# 2. Load the Base Model with the config
17# Use device_map="auto" for efficient loading with quantization
18# Use torch_dtype=torch.bfloat16 for Llama models with bnb
19model = AutoModelForCausalLM.from_pretrained(
20 base_model_name_or_path,# The PEFT adapter ID
21 quantization_config=bnb_config,
22 torch_dtype=torch.bfloat16,
23 device_map="cuda",
24)
25
26tokenizer=AutoTokenizer.from_pretrained(base_model_name_or_path)
27system_prompt="""
28 You are a reasonable expert who thinks and answer the users question.
29 Before respond first think and create a chain of thoughts in your mind.
30 Then respond to the client.
31 Your chain of thought and reflection must be in <thinking>..</thinking> format and your respond
32 should be in the <output>..</output> format.
33 """
34
35 messages = [
36 {'role':'system','content':system_prompt},
37 {"role": "user", "content":message},
38
39 ]
40
41 inputs = tokenizer.apply_chat_template(
42 messages,
43 tokenize = True,
44 add_generation_prompt = True, # Must add for generation
45 return_tensors = "pt",).to("cuda")
46 inputs_shape=inputs['input_ids'].shape[1]
47 with torch.no_grad():
48 output=model.generate(**inputs, max_new_tokens =2048,
49 use_cache = True, temperature = 0.5, min_p = 0.9)
50
51