Views
No views yet
pip install -q sentencepiece pip install -q transformers pip install -q accelerate pip install --upgrade -q bitsandbytes 1
2import torch
3from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
4
5model_path = "Neurai/llama7b"
6tokenizer = AutoTokenizer.from_pretrained(model_path)
7model = AutoModelForCausalLM.from_pretrained(
8 "Neurai/llama7b",
9 # load_in_8bit=True,
10 torch_dtype=torch.bfloat16,
11 low_cpu_mem_usage=True,
12 device_map="auto",
13 )
14model.eval()
15print('model loaded')
16
17SYS_PROMPT = "زرافه چند سال عمر میکند؟"
18
19def response_generate(input_prompt):
20 input_ids = tokenizer(input_prompt, return_tensors="pt")
21 outputs = model.generate(
22 inputs=input_ids["input_ids"].to("cuda"),
23 attention_mask=input_ids["attention_mask"].to("cuda"),
24 do_sample=True,
25 temperature=0.3,
26 top_k=50,
27 top_p=0.9,
28 max_new_tokens=512,
29 eos_token_id=tokenizer.eos_token_id,
30 pad_token_id=tokenizer.pad_token_id
31 )
32 response = tokenizer.batch_decode(outputs, skip_special_tokens=True)[0]
33 return response
34
35print(response_generate(f"{SYS_PROMPT}"))