Views
No views yet
1from transformers import AutoTokenizer, AutoModelForCausalLM
2import torch
3
4model_id = "mzbac/llama-3-8B-grammar-hf"
5tokenizer = AutoTokenizer.from_pretrained(model_id)
6model = AutoModelForCausalLM.from_pretrained(
7 model_id,
8 torch_dtype=torch.bfloat16,
9 device_map="auto",
10
11)
12
13messages = [
14 {
15 "role": "system",
16 "content": "Please correct, polish, or translate the text delimited by triple backticks to standard English.",
17 },
18]
19messages.append({"role": "user", "content":"Text=```neither 经理或员工 has been informed about the meeting```"})
20
21input_ids = tokenizer.apply_chat_template(
22 messages,
23 add_generation_prompt=True,
24 return_tensors="pt"
25).to(model.device)
26
27terminators = [
28 tokenizer.eos_token_id,
29 tokenizer.convert_tokens_to_ids("<|eot_id|>")
30]
31
32outputs = model.generate(
33 input_ids,
34 max_new_tokens=256,
35 eos_token_id=terminators,
36 do_sample=True,
37 temperature=0.1,
38)
39response = outputs[0]
40print(tokenizer.decode(response))
41
42# <|begin_of_text|><|start_header_id|>system<|end_header_id|>
43
44# Please correct, polish, or translate the text delimited by triple backticks to standard English.<|eot_id|><|start_header_id|>user<|end_header_id|>
45
46# Text=```neither 经理或员工 has been informed about the meeting```<|eot_id|><|start_header_id|>assistant<|end_header_id|>
47
48# Output=Neither the manager nor the employees have been informed about the meeting.<|eot_id|>