Views
No views yet
1from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
2import torch
3
4model_name_or_id = "MehdiHosseiniMoghadam/AVA-Qwen1.5-7B-Chat"
5model = AutoModelForCausalLM.from_pretrained(model_name_or_id, torch_dtype=torch.float16, device_map="auto", low_cpu_mem_usage=True, load_in_8bit=True)
6tokenizer = AutoTokenizer.from_pretrained(model_name_or_id)
7
8prompt = ''
9
10prompt = f"### Human:{prompt}\n### Assistant:"
11
12
13inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
14
15generation_config = GenerationConfig(
16 do_sample=True,
17 top_k=1,
18 temperature=0.01,
19 max_new_tokens=90,
20 pad_token_id=tokenizer.eos_token_id
21)
22
23
24outputs = model.generate(**inputs, generation_config=generation_config)
25print(tokenizer.decode(outputs[0], skip_special_tokens=True))
26
27
