Views
No views yet
1pip3 install git+https://github.com/intel/auto-round.git
2pip3 install auto-gptq1from auto_round import AutoRoundConfig ##must import for autoround format
2from transformers import AutoModelForCausalLM, AutoTokenizer
3import torch
4
5quantized_model_dir = "OPEA/MiniMax-Text-01-int4-sym-inc-preview"
6
7tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir, trust_remote_code=True)
8model = AutoModelForCausalLM.from_pretrained(quantized_model_dir, trust_remote_code=True, torch_dtype=torch.bfloat16,
9 device_map="auto")
10
11
12def forward_hook(module, input, output):
13 return torch.clamp(output, -65504, 65504).to(torch.bfloat16)
14
15
16def register_fp16_hooks(model):
17 for name, module in model.named_modules():
18 if "QuantLinear" in module.__class__.__name__ or isinstance(module, torch.nn.Linear):
19 module.register_forward_hook(forward_hook)
20
21
22register_fp16_hooks(model)
23tokenizer.pad_token = tokenizer.eos_token
24
25prompts = [
26 "为什么企鹅没有被北极熊吃掉？",
27 "树枝上有十只鸟，如果你射杀了一只，还剩下几只？请用中文回答",
28 "How many r in strawberry.",
29 "There is a girl who likes adventure,",
30 "hello"
31]
32
33texts = []
34for prompt in prompts:
35 messages = [
36 {"role": "system", "content": [{"type": "text",
37 "text": "You are a helpful assistant created by MiniMax based on MiniMax-Text-01 model."}]},
38 {"role": "user", "content": [{"type": "text", "text": prompt}]},
39 ]
40 text = tokenizer.apply_chat_template(
41 messages,
42 tokenize=False,
43 add_generation_prompt=True
44 )
45 texts.append(text)
46inputs = tokenizer(texts, return_tensors="pt", padding=True, truncation=True, padding_side='left')
47
48outputs = model.generate(
49 input_ids=inputs["input_ids"].to(model.device),
50 attention_mask=inputs["attention_mask"].to(model.device),
51 max_new_tokens=512,
52 num_return_sequences=1,
53 do_sample=False,
54 eos_token_id=200020,
55)
56generated_ids = [
57 output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
58]
59
60decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
61
62for i, prompt in enumerate(prompts):
63 input_id = inputs
64 print(f"Prompt: {prompt}")
65 print(f"Generated: {decoded_outputs[i]}")
66 print("-" * 50)
67
68
69"""
70Prompt: 为什么企鹅没有被北极熊吃掉？
71Generated: ### 1. **地理分布差异**
72 - **企鹅**：主要生活在**南半球**，例如**南极洲**。在南极洲，企鹅没有天敌，因为这里的环境非常恶劣，食物资源有限，动物数量也有限，企鹅是这里的顶级掠食者之一。
73 - **北极熊**：主要生活在**北半球**，例如**北极地区**。北极熊是北极地区的顶级掠食者之一，它们以海豹等动物为食。
74 - **结论**：由于**地理分布**的差异，**企鹅和北极熊**在自然界中**无法相遇**，因此**北极熊无法吃掉企鹅**。
75
76### 2. **人为因素**
77 - **动物园或水族馆**：在**人为因素**的影响
78--------------------------------------------------
79Prompt: 树枝上有十只鸟，如果你射杀了一只，还剩下几只？请用中文回答
80Generated: 让我一步步思考这个问题：
81
821. 首先,树枝上有10只鸟
832. 射杀1只后,还剩9只
843. 但实际上,当枪声响起,其他鸟会因惊吓而飞走
854. 所以,当射杀1只后,树上不会剩下任何鸟
86
87因此,答案是:0只
88
89因为鸟会因枪声而飞走,不会继续停留在树上。
90--------------------------------------------------
91Prompt: How many r in strawberry.
92Generated: Let me solve this step by step.
93
941. First, let me count the r's in "strawberry" as I say it
95 * s (not r)
96 * t (not r)
97 * r (1st r)
98 * a (not r)
99 * w (not r)
100 * b (not r)
101 * b (not r)
102 * e (not r)
103 * r (2nd r)
104 * r (3rd r)
105 * y (not r)
106
1072. Counting the r's: 3 r's
108
109Therefore, there is 3 r in strawberry.
110
111The answer is 3.
112--------------------------------------------------
113Prompt: There is a girl who likes adventure,
114Generated: There is a girl who likes adventure, and her name is Emily. Emily has always been drawn to the thrill of the unknown, the excitement of stepping into uncharted territory. Here is a story about
115--------------------------------------------------
116Prompt: hello
117Generated: Hello! How can I assist you today?
118--------------------------------------------------
119"""1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3
4model_name = "MiniMaxAI/MiniMax-Text-01"
5tokenizer = AutoTokenizer.from_pretrained(model_name)
6model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.bfloat16)
7
8fp_layers = [f"model.layers.{i}.block_sparse_moe.gate" for i in range(model.config.num_hidden_layers)]
9layer_config = {}
10for fp_layer in fp_layers:
11 layer_config[fp_layer] = {"bits": 16}
12
13device_map = {}
14for i in range(32):
15 key = fr"model\.layers\.\d+\.block_sparse_moe\.experts\.{str(i)}\..*$"
16 if i < 14:
17 device_map[key] = 0
18 else:
19 device_map[key] = 1
20
21
22from auto_round import AutoRound
23
24autoround = AutoRound(model=model, tokenizer=tokenizer, layer_config=layer_config, device_map=device_map,
25 batch_size=1,gradient_accumulate_steps=4, seqlen=512)
26autoround.quantize()
27autoround.save_quantized(format="auto_round", output_dir="tmp_autoround")
28