Views
No views yet
unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bitshehrozrafaqat/football-llama31-8b-qlorashehrozrafaqat/global-football-assistant-datasetSFTTrainer + QLoRAbase_model_eval.jsonlfinetuned_model_eval.jsonlpromptexpected_answermodel_answersubjectcategory1import torch
2from peft import PeftModel
3from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
4
5base_model_id = "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit"
6adapter_id = "shehrozrafaqat/football-llama31-8b-qlora"
7
8bnb_config = BitsAndBytesConfig(load_in_4bit=True)
9
10tokenizer = AutoTokenizer.from_pretrained(adapter_id)
11base_model = AutoModelForCausalLM.from_pretrained(
12 base_model_id,
13 quantization_config=bnb_config,
14 device_map="auto",
15 torch_dtype=torch.float16,
16)
17model = PeftModel.from_pretrained(base_model, adapter_id)
18model.eval()
19
20messages = [
21 {"role": "system", "content": "You are a careful football knowledge assistant."},
22 {"role": "user", "content": "Explain goal difference in football."},
23]
24
25prompt = tokenizer.apply_chat_template(
26 messages,
27 tokenize=False,
28 add_generation_prompt=True,
29)
30inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
31
32with torch.no_grad():
33 output_ids = model.generate(
34 **inputs,
35 max_new_tokens=120,
36 do_sample=False,
37 pad_token_id=tokenizer.eos_token_id,
38 )
39
40answer_ids = output_ids[0][inputs["input_ids"].shape[-1]:]
41print(tokenizer.decode(answer_ids, skip_special_tokens=True).strip())