Views
No views yet
1alpaca_prompt = "### Text: {} ### Summary: {}"
2FastLanguageModel.for_inference(model)
3inputs = tokenizer(
4[
5 alpaca_prompt.format(
6 "", # text to summarize
7 "", # output - leave this blank for generation!
8 )
9], return_tensors = "pt").to("cuda")
10
11outputs = model.generate(**inputs, max_new_tokens = 64, use_cache = True)
12tokenizer.batch_decode(outputs)