from transformers import AutoModelForSequenceClassification, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel
import torch
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True, bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.float16,
)
base = AutoModelForSequenceClassification.from_pretrained(
    "google/gemma-2-2b", num_labels=2,
    quantization_config=bnb_config, torch_dtype=torch.float16, device_map="auto",
)
tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-2b")
base.config.pad_token_id = tokenizer.pad_token_id
model = PeftModel.from_pretrained(base, "Mohammadeeu20/besstie-sarcasm-gemma2b-lora-en-au")

Training details

LoRA r=16, α=32, dropout=0.05, target modules q_proj/k_proj/v_proj/o_proj
4-bit NF4 quantization with double-quant
4 epochs, lr=2e-4, paged_adamw_8bit
Class-weighted cross-entropy

Coursework context

Trained for COMM061 NLP coursework (University of Surrey, 2026) using the surrey-nlp/BESSTIE-CW-26 dataset.