from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel

# Load base model and adapter
base_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    device_map="auto",
    torch_dtype="auto"
)
model = PeftModel.from_pretrained(base_model, "YOUR_USERNAME/squad-v2-llama3-lora-improved")
tokenizer = AutoTokenizer.from_pretrained("YOUR_USERNAME/squad-v2-llama3-lora-improved")

# Format prompt
prompt = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>

You are an EXPERT answer-span extractor. Extract the SHORTEST EXACT SPAN from the context that answers the question. Output [] if unanswerable.<|eot_id|><|start_header_id|>user<|end_header_id|>

Context: The Normans were the people who gave their name to Normandy, a region in France.
Question: In what country is Normandy located?<|eot_id|><|start_header_id|>assistant<|end_header_id|>

"""

inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=50, do_sample=False)
answer = tokenizer.decode(outputs[0], skip_special_tokens=True)
# Output: "France"

Key Improvements

Structured Prompt: Uses a system prompt with few-shot examples
EOS Token Training: Model learns to stop after generating the answer
Short Answer Extraction: Trained to output minimal spans

Limitations

Optimized for English text only
Best for factoid questions with explicit answers in context
May struggle with very long contexts (>1000 tokens)

Training Data

Trained on the full SQuAD 2.0 dataset (130,319 training examples).

Author

Finetuned with ❤️ by Vishakan Umapathy

ivishakan/squad-v2-llama3-lora-improved

SQuAD v2 Answer Extractor (LLaMA 3 8B + LoRA)

Model Description

Training Configuration

Usage

Key Improvements

Limitations

Training Data

Author

ivishakan/
squad-v2-llama3-lora-improved