vericudebuget/

qwen-2.5-0.5b-finetuned-Bible

from unsloth import FastLanguageModel

model, tokenizer = FastLanguageModel.from_pretrained(
        model_name = "lora_model1", # YOUR MODEL YOU USED FOR TRAINING
        max_seq_length = max_seq_length,
        dtype = dtype,
        load_in_4bit = load_in_4bit,
    )

# Switch to inference mode using FastLanguageModel.for_inference(model)
FastLanguageModel.for_inference(model) # Fix: Use for_inference instead of for_training

alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

### Instruction:
{}

### Input:
{}

### Response:
{}"""

# Removed redundant FastLanguageModel.for_training(model) call

inputs = tokenizer(
[
    alpaca_prompt.format(
        "", # instruction
        "Who is Jesus", # input
        "", # output - leave this blank for generation!
    )
], return_tensors = "pt").to("cuda")


from transformers import TextStreamer
text_streamer = TextStreamer(tokenizer)
_ = model.generate(**inputs, streamer = text_streamer, max_new_tokens = 64)