GeneralRincewind/

SonnetLlama

from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("GeneralRincewind/SonnetLlama")
model = AutoModelForCausalLM.from_pretrained("GeneralRincewind/SonnetLlama")

#### Generate text
from transformers import TextStreamer
tokenized_text = tokenizer("", return_tensors="pt",  truncation=True)
input_ids = tokenized_text.input_ids
streamer = TextStreamer(tokenizer)
model.eval()
full_completion = model.generate(inputs=tokenized_text["input_ids"].to("cuda"),
    attention_mask=tokenized_text["attention_mask"].to("cuda"),
    temperature=0.9,
     top_k=80,
     top_p=0.65,
    do_sample=True,
    streamer=streamer,                           
    num_beams=1,
    max_new_tokens=500,
    eos_token_id=tokenizer.eos_token_id,
    pad_token_id=tokenizer.pad_token_id,
    repetition_penalty=1)

decoded_text = tokenizer.decode(full_completion[0])
print(decoded_text)