Views
No views yet
1from unsloth import FastLanguageModel
2from transformers import TextStreamer
3 model, tokenizer = FastLanguageModel.from_pretrained(
4 model_name = "lora_model", # YOUR MODEL YOU USED FOR TRAINING
5 max_seq_length = max_seq_length,
6 dtype = dtype,
7 load_in_4bit = load_in_4bit,
8 )
9FastLanguageModel.for_inference(model)
10inputs = tokenizer(
11[
12 "Official Title: Randomized Trial of Usual Care vs. Specialized, Phase-specific Care for Youth at Risk for Psychosis"
13], return_tensors = "pt").to("cuda")
14
15text_streamer = TextStreamer(tokenizer, skip_prompt = True)
16_ = model.generate(input_ids = inputs.input_ids, attention_mask = inputs.attention_mask,
17 streamer = text_streamer, max_new_tokens = 2048, pad_token_id = tokenizer.eos_token_id)