Views
No views yet
1 model, tokenizer = FastLanguageModel.from_pretrained(
2 model_name = "model",
3 max_seq_length = 2048,
4 dtype = dtype,
5 load_in_4bit = load_in_4bit,
6 )
7
8 alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
9 ## Instruction:
10 {}
11
12 ## Input:
13 {}
14
15 ## Response:
16 {}
17 """
18
19 FastLanguageModel.for_inference(model)
20
21 inputs = tokenizer(
22 [
23 alpaca_prompt.format(
24 test_questions[0],
25 "",
26 "",
27 )
28 ],
29 return_tensors="pt"
30 ).to("cuda")
31
32 outputs = model.generate(
33 **inputs,
34 max_new_tokens=4096,
35 use_cache=True
36 )
37
38 decoded_outputs = tokenizer.batch_decode(outputs, skip_special_tokens=True)
39 print(decoded_outputs[0])