Views
No views yet
1from transformers import AutoTokenizer,GenerationConfig,AutoConfig,GenerationConfig,AutoModelForCausalLM
2import torch
3from transformers import AutoTokenizer
4model_path = "model_path"
5model = AutoModelForCausalLM.from_pretrained(model_path,trust_remote_code = True)
6tokenizer = AutoTokenizer.from_pretrained(model_path)
7
8_inputs = """your input"""
9
10gen_conf = GenerationConfig(
11 num_beams=1,
12 do_sample=False,
13 max_length=128,
14 max_new_tokens=128,
15 no_repeat_ngram_size=4,
16 eos_token_id=tokenizer.eos_token_id,
17 pad_token_id=tokenizer.pad_token_id,
18)
19
20tokend = tokenizer.encode_plus(text=_inputs, add_special_tokens=False)
21input_ids_lengths = len(tokend.input_ids)
22print(_inputs)
23input_ids = torch.LongTensor([tokend.input_ids])
24
25outputs = model.generate(
26 inputs=input_ids,
27 generation_config=gen_conf,
28)
29
30outs = tokenizer.decode(outputs[0][input_ids_lengths:],skip_special_tokens=False)
31print(outs)
32