AiLab-IMCS-UL/

Gemma3-12B-Instruct-LVportals-15K

    import os
    os.environ['WANDB_DISABLED'] = 'true'

    from unsloth import FastModel
    from unsloth import get_chat_template

    max_seq_length = 2048
    dtype = None
    load_in_4bit = True

    model, tokenizer = FastModel.from_pretrained(
        model_name = 'lora',
        max_seq_length=max_seq_length,
        dtype=dtype,
        load_in_4bit=load_in_4bit
    )

    tokenizer = get_chat_template(
        tokenizer,
        chat_template = "gemma-3",
    )

    prompt_style = """<start_of_turn>user
    {}<end_of_turn>
    <start_of_turn>model
    {}"""

    FastModel.for_inference(model)  # Unsloth has 2x faster inference!

    prompt = "Kas ir rakstīts civillikuma 594. pantā?"

    inputs = tokenizer([prompt_style.format(prompt, "")], return_tensors="pt").to("cuda")

    outputs = model.generate(
        input_ids=inputs.input_ids,
        attention_mask=inputs.attention_mask,
        max_new_tokens=2048,
        use_cache=True,
    )
    response = tokenizer.batch_decode(outputs)
    answer = response[0].split('<start_of_turn>model\n')[-1].replace('<end_of_turn>','')

    print(answer)