from unsloth import FastLanguageModel
import torch

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name = "binaryecheos/ai-tutor-llama3-lora",
    max_seq_length = 1024,
    load_in_4bit = True,
)

FastLanguageModel.for_inference(model)

Generate Response

Developed by: binaryecheos
License: apache-2.0
Finetuned from model: unsloth/llama-3.2-3b-unsloth-bnb-4bit

python

system_prompt = (
    "You are an expert AI tutor helping university students "
    "understand Artificial Intelligence concepts clearly and accurately."
)

user_prompt = "Explain A* algorithm in detail."

prompt = f"""<|begin_of_text|><|start_header_id|>system<|end_header_id|>

{system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>

{user_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>

"""

inputs = tokenizer(
    prompt,
    return_tensors="pt"
).to("cuda")

eot_token = tokenizer.convert_tokens_to_ids("<|eot_id|>")

outputs = model.generate(
    **inputs,
    max_new_tokens = 500,
    temperature = 0.4,
    top_p = 0.9,
    repetition_penalty = 1.15,
    do_sample = True,
    eos_token_id = eot_token,
    pad_token_id = tokenizer.eos_token_id,
)

response = tokenizer.decode(
    outputs[0][inputs["input_ids"].shape[1]:],
    skip_special_tokens=True
)

print(response)

This llama model was trained 2x faster with Unsloth