Views
No views yet
<|im_end|>.1from transformers import AutoModelForCausalLM, AutoTokenizer
2tok = AutoTokenizer.from_pretrained("REPO_ID")
3model = AutoModelForCausalLM.from_pretrained("REPO_ID", torch_dtype="bfloat16").cuda()
4ids = tok(query, return_tensors="pt").input_ids.cuda()
5out = model.generate(ids, max_new_tokens=300, do_sample=False)
6print(tok.decode(out[0][ids.shape[1]:], skip_special_tokens=True))