from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import PeftModel
import torch

BASE_MODEL = "Qwen/Qwen2.5-7B-Instruct"
LORA_PATH  = "lifatsastain/teach_lora1"

quant_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_use_double_quant=True,
    bnb_4bit_compute_dtype=torch.bfloat16,
)

model = AutoModelForCausalLM.from_pretrained(
    BASE_MODEL, quantization_config=quant_config, device_map="auto"
)
model = PeftModel.from_pretrained(model, LORA_PATH)
model.eval()

tokenizer = AutoTokenizer.from_pretrained(LORA_PATH)

SYSTEM = (
    "You are an ML tutor teaching CS students who know coding but not ML. "
    "Always start with an intuitive analogy, build up to the concept, "
    "and end with a practice question. Be encouraging and patient."
)

messages = [
    {"role": "system", "content": SYSTEM},
    {"role": "user",   "content": "What is gradient descent?"},
]
text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(text, return_tensors="pt").to(model.device)

with torch.no_grad():
    output_ids = model.generate(**inputs, max_new_tokens=512, temperature=0.7, top_p=0.9,
                                 do_sample=True, pad_token_id=tokenizer.eos_token_id)

new_tokens = output_ids[0][inputs["input_ids"].shape[-1]:]
print(tokenizer.decode(new_tokens, skip_special_tokens=True))

IF YOU WANT TALK IN LONG CONVERSATION

python

from peft import PeftModel
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
import bitsandbytes
BASE_MODEL = "Qwen/Qwen2.5-7B-Instruct"
LORA_PATH  = "lifatsastain/teach_lora1"

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
)

tokenizer = AutoTokenizer.from_pretrained(LORA_PATH)

base_model = AutoModelForCausalLM.from_pretrained(
    BASE_MODEL,
    quantization_config=bnb_config,
    device_map="auto",
)
model = PeftModel.from_pretrained(base_model, LORA_PATH)
model.eval()

SYSTEM = (
    "You are an ML tutor teaching CS students who know coding but not ML. "
    "Always start with an intuitive analogy, build up to the concept, "
    "and end with a practice question. Be encouraging and patient."
)

conversation_history = []

def chat(user_message):
    conversation_history.append({"role": "user", "content": user_message})

    messages = [{"role": "system", "content": SYSTEM}] + conversation_history

    text = tokenizer.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=True
    )
    inputs = tokenizer(text, return_tensors="pt").to(model.device)

    with torch.no_grad():
        output_ids = model.generate(
            **inputs,
            max_new_tokens=1024,
            temperature=0.7,
            top_p=0.9,
            do_sample=True,
            pad_token_id=tokenizer.eos_token_id
        )

    new_tokens = output_ids[0][inputs["input_ids"].shape[-1]:]
    response = tokenizer.decode(new_tokens, skip_special_tokens=True)

    conversation_history.append({"role": "assistant", "content": response})
    return response

print("ML Tutor ready! Type 'quit' to exit, 'reset' to clear history.\n")

while True:
    user_input = input("You: ").strip()
    if not user_input:
        continue
    if user_input.lower() == "quit":
        print("Bye!")
        break
    if user_input.lower() == "reset":
        conversation_history.clear()
        print("Conversation history cleared.\n")
        continue
    response = chat(user_input)
    print(f"\nTutor: {response}\n")

'''



## Training Details

| Parameter              | Value                          |
|------------------------|-------------------------------|
| Base model             | Qwen2.5-7B-Instruct            |
| LoRA rank (r)          | 16                             |
| LoRA alpha             | 32                             |
| LoRA dropout           | 0.05                           |
| Target modules         | q_proj, k_proj, v_proj, o_proj |
| Training epochs        | 1                              |
| Learning rate          | 2e-4                           |
| Batch size             | 1 (grad accum 16)              |
| Max sequence length    | 512                            |
| Quantization           | 4-bit NF4                      |
| Optimizer              | paged_adamw_8bit               |
|----------------------------------------------------------

### Framework Versions
- transformers: 5.3.0
- bitsandbytes: 0.49.2
- peft: 0.18.1
- torch: 2.10.0+cu126
- trl: 0.29.0
- datasets: 4.7.0