model_id = "Unbabel/TowerInstruct-7B-v0.2"
tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.padding_side="left"
padding="longest"
model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto")
model.load_adapter("skoneru/iwslt_mt_ende")

<|im_start|>user\nTranslate the sentence from English into German.
English:
{src_sentence}
German:<|im_end|>\n<|im_start|>assistant
{llm to generate}

src_sent = "Welcome to the first lecture"
prefix = "<|im_start|>user\nTranslate the sentence from English into German.\nEnglish: "
suffix = "\nGerman:<|im_end|>\n<|im_start|>assistant\n"
prompt = [prefix + src_sent + suffix]
inputs = tokenizer(prompt, return_tensors="pt", padding=True, add_special_tokens=False).to(model.device)
num_beams=5

output = model.generate(**inputs, num_beams=num_beams, max_new_tokens=256, return_dict_in_generate=True, early_stopping=True, do_sample=False)
hyps = tokenizer.batch_decode(output.sequences[:,inputs.input_ids.shape[1]:], skip_special_tokens=True)
print(hyps)