Views
No views yet
1pip install git+https://github.com/huggingface/transformers --upgrade
2pip install torch accelerate bitsandbytes flash_attn1from transformers import AutoTokenizer, AutoModelForCausalLM, MixtralForCausalLM
2import torch
3
4model_name_or_path = "mmnga/Mixtral-Fusion-4x7B-Instruct-v0.1"
5
6tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
7model = MixtralForCausalLM.from_pretrained(model_name_or_path, load_in_8bit=True)
8
9# set num_experts_per_tok 1 or 2 ?
10model.config.num_experts_per_tok = 2
11
12# message
13messages = [
14 {"role": "user", "content": "Tell me what's for dinner tonight."},
15]
16
17with torch.no_grad():
18 token_ids = tokenizer.apply_chat_template(messages, return_tensors="pt")
19 output_ids = model.generate(
20 token_ids.to(model.device),
21 temperature=0.5,
22 do_sample=True,
23 top_p=0.95,
24 top_k=40,
25 max_new_tokens=128,
26 repetition_penalty=1.5
27 )
28output = tokenizer.decode(output_ids[0][token_ids.size(1) :])
29print(output)
30