Views
No views yet
1import torch
2from unsloth import FastLanguageModel
3from unsloth.chat_templates import get_chat_template
4
5model_save_path = "path to the download folder" #the hugging face folder path pulled.
6
7model, tokenizer = FastLanguageModel.from_pretrained(
8 model_name=model_save_path,
9 max_seq_length=4096,
10 load_in_4bit=True,
11)
12FastLanguageModel.for_inference(model) # Enable native 2x faster inference
13
14tokenizer = get_chat_template(
15 tokenizer,
16 chat_template="llama-3", # use the llama-3 template
17 mapping={"role": "from", "content": "value", "user": "human", "assistant": "gpt"}, # mapping the messages.
18)
19
20messages = [{"from": "human", "value": "your prompt"}]#add your prompt here as human
21inputs = tokenizer.apply_chat_template(
22 messages,
23 tokenize=True,
24 add_generation_prompt=True, # Must add for generation
25 return_tensors="pt",
26).to("cuda")
27
28outputs = model.generate(input_ids=inputs, max_new_tokens=2048, use_cache=True)
29response = tokenizer.batch_decode(outputs, skip_special_tokens=True)
30print(response)