Views
No views yet
1pip install sentencepiece
2pip install transformers>=4.28.01import torch
2import transformers
3from transformers import LlamaTokenizer, LlamaForCausalLM
4
5def generate_prompt(text):
6 return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.
7
8### Instruction:
9{text}
10
11### Response:"""
12
13
14tokenizer = LlamaTokenizer.from_pretrained('minlik/chinese-alpaca-pro-33b-merged')
15model = LlamaForCausalLM.from_pretrained('minlik/chinese-alpaca-pro-33b-merged').half().to('cuda')
16model.eval()
17
18text = '第一个登上月球的人是谁？'
19prompt = generate_prompt(text)
20input_ids = tokenizer.encode(prompt, return_tensors='pt').to('cuda')
21
22
23with torch.no_grad():
24 output_ids = model.generate(
25 input_ids=input_ids,
26 max_new_tokens=128,
27 temperature=1,
28 top_k=40,
29 top_p=0.9,
30 repetition_penalty=1.15
31 ).cuda()
32output = tokenizer.decode(output_ids[0], skip_special_tokens=True)
33print(output.replace(prompt, '').strip())