Views
No views yet
1from unsloth import FastLanguageModel
2max_seq_length = 2048
3dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
4load_in_4bit = True
5model, tokenizer = FastLanguageModel.from_pretrained(
6 model_name = "imranali291/Human-Like-Llama-3.2-1B3B-Conversational-LORA",
7 max_seq_length = max_seq_length,
8 dtype = dtype,
9 load_in_4bit = load_in_4bit,
10)
11FastLanguageModel.for_inference(model) # Enable native 2x faster inference
12
13messages = [
14 {"role": "user", "content": "Hi!, how are you?"},
15]
16inputs = tokenizer.apply_chat_template(
17 messages,
18 tokenize = True,
19 add_generation_prompt = True, # Must add for generation
20 return_tensors = "pt",
21).to("cuda")
22
23from transformers import TextStreamer
24text_streamer = TextStreamer(tokenizer)
25_ = model.generate(input_ids = inputs, streamer = text_streamer, max_new_tokens = 128, use_cache = True)
26