Views
No views yet
1from unsloth.chat_templates import get_chat_template
2from unsloth import FastLanguageModel
3
4# Get the chat template
5tokenizer = get_chat_template(
6 tokenizer,
7 chat_template="llama-3.1",
8)
9model = "MateoRov/Llama3.2-3b-SFF-Infinity-MateoRovere"
10
11# Enable native 2x faster inference
12FastLanguageModel.for_inference(model)
13
14# Define the input message
15messages = [
16 {"role": "user", "content": "Continue the Fibonacci sequence: 1, 1, 2, 3, 5, 8,"},
17]
18
19# Prepare the inputs
20inputs = tokenizer.apply_chat_template(
21 messages,
22 tokenize=True,
23 add_generation_prompt=True, # Must add for generation
24 return_tensors="pt",
25).to("cuda")
26
27# Generate the output
28outputs = model.generate(
29 input_ids=inputs,
30 max_new_tokens=64,
31 use_cache=True,
32 temperature=1.5,
33 min_p=0.1,
34)
35
36# Decode the outputs
37result = tokenizer.batch_decode(outputs)
38print(result)1
2from unsloth.chat_templates import get_chat_template
3from unsloth import FastLanguageModel
4from transformers import TextStreamer
5
6model = "MateoRov/Llama3.2-3b-SFF-Infinity-MateoRovere"
7
8# Enable native 2x faster inference
9FastLanguageModel.for_inference(model)
10
11# Get the chat template
12tokenizer = get_chat_template(
13 tokenizer,
14 chat_template="llama-3.1",
15)
16
17# Define the input message
18messages = [
19 {"role": "user", "content": "Continue the Fibonacci sequence: 1, 1, 2, 3, 5, 8,"},
20]
21
22# Prepare the inputs
23inputs = tokenizer.apply_chat_template(
24 messages,
25 tokenize=True,
26 add_generation_prompt=True, # Must add for generation
27 return_tensors="pt",
28).to("cuda")
29
30# Initialize the text streamer
31text_streamer = TextStreamer(tokenizer, skip_prompt=True)
32
33# Generate the output token by token
34_ = model.generate(
35 input_ids=inputs,
36 streamer=text_streamer,
37 max_new_tokens=128,
38 use_cache=True,
39 temperature=1.5,
40 min_p=0.1,
41)