Views
No views yet
1from transformers import AutoModelForCausalLM, AutoTokenizer
2import transformers
3from auto_round import AutoRound, AutoRoundConfig
4
5import torch
6
7quantized_model_dir = "Intel/Kimi-K2-Instruct-int4-mixed-AutoRound-cpu"
8
9model = AutoModelForCausalLM.from_pretrained(
10 quantized_model_dir,
11 torch_dtype=torch.bfloat16,
12 device_map="cpu",
13)
14tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir, trust_remote_code=True)
15prompts = [
16 "9.11和9.8哪个数字大",
17 "strawberry中有几个r?",
18 "There is a girl who likes adventure,",
19 "Please give a brief introduction of Moonshot AI",
20]
21
22texts=[]
23for prompt in prompts:
24 messages = [
25 {"role": "system", "content": "You are Kimi, an AI assistant created by Moonshot AI."},
26 {"role": "user", "content": [{"type": "text", "text":prompt}]}
27 ]
28 text = tokenizer.apply_chat_template(
29 messages,
30 tokenize=False,
31 add_generation_prompt=True
32 )
33 texts.append(text)
34inputs = tokenizer(texts, return_tensors="pt", padding=True, truncation=True)
35
36outputs = model.generate(
37 input_ids=inputs["input_ids"].to(model.device),
38 attention_mask=inputs["attention_mask"].to(model.device),
39 max_length=200, ##change this to align with the official usage
40 num_return_sequences=1,
41 do_sample=False ##change this to align with the official usage
42)
43generated_ids = [
44 output_ids[len(input_ids):] for input_ids, output_ids in zip(inputs["input_ids"], outputs)
45]
46
47decoded_outputs = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
48
49for i, prompt in enumerate(prompts):
50 input_id = inputs
51 print(f"Prompt: {prompt}")
52 print(f"Generated: {decoded_outputs[i]}")
53 print("-" * 50)
54"""
55Prompt: 9.11和9.8哪个数字大
56Generated: ### 第一步：理解题目
57
58首先，我需要明确题目在问什么。题目给出了两个数字：9.11和9.8，问哪一个更大。这看起来是一个简单的数值比较问题。
59
60### 第二步：数字的表示
61
62这两个数字都是小数，即带有小数部分的数字。小数由整数部分和小数部分组成，小数点左边是整数部分，右边是小数部分。
63
64- 9.11：整数部分是9，小数部分是11。
65- 9.8：整数部分是9，小数部分是8。
66
67### 第三步：比较整数部分
68
69首先比较两个数的整数部分：
70
71- 9.11的整数部分是9。
72- 9.8的整数部分也是9。
73
74整数部分相同，因此需要比较小数部分。
75
76### 第四步：比较小数部分
77
78小数部分的比较
79--------------------------------------------------
80Prompt: strawberry中有几个r?
81Generated: ### 问题重述
82我们需要计算单词 "strawberry" 中有多少个字母 "r"。
83
84### 步骤分解
851. **写出单词**：首先，将单词 "strawberry" 完整地写出来。
862. **逐个字母检查**：从左到右，逐个字母查看是否是 "r"（注意大小写，但这里都是小写）。
873. **计数**：每遇到一个 "r"，就增加计数器。
88
89### 详细检查
90让我们将 "strawberry" 拆分开来：
91
92字母位置及字母：
931. s
942. t
953. r
964. a
975. w
986. b
997. e
1008. r
1019. r
10210. y
103
104现在，我们检查每个字母是否为 "r"：
105
106-
107--------------------------------------------------
108Prompt: There is a girl who likes adventure,
109Generated: There is a girl who likes adventure,
110so she ties her shoes with sunrise instead of laces,
111lets the wind pick the next city,
112and trades her shadow for a passport stamp.
113
114She keeps her memories in mason jars—
115one holds the scent of monsoon in Mumbai,
116another the hush of Icelandic snow.
117When homesick, she unscrews a lid,
118inhales, and is gone again.
119
120She once outran her own name
121somewhere between Marrakesh and the moon,
122answering only to “Hey, you with the constellations in your hair.”
123Maps are her love letters;
124she folds them into paper boats
125and sails them down hotel bathtubs,
126whispering, *Find me where the water ends.*
127--------------------------------------------------
128Prompt: Please give a brief introduction of Moonshot AI
129Generated: Moonshot AI is a Chinese artificial-intelligence company founded in 2023 and headquartered in Beijing. Focused on large-scale language models and related products, it released its first model, Kimi, in October 2023 and has since launched upgraded versions such as Kimi 1.5. The company closed a US$1 billion funding round in early 2024 that valued it at about US$2.5 billion, making it one of China’s best-funded AI start-ups.
130--------------------------------------------------
131
132"""1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer
3import transformers
4
5model_name = "Kimi-K2-Instruct-BF16"
6
7tokenizer = AutoTokenizer.from_pretrained(model_name,trust_remote_code=True)
8model = AutoModelForCausalLM.from_pretrained(model_name,device_map="cpu", torch_dtype="auto",trust_remote_code=True)
9
10layer_config = {}
11for n, m in model.named_modules():
12 if isinstance(m, torch.nn.Linear):
13 if "expert" in n or "shared_experts" in n:
14 layer_config[n] = {"bits": 4}
15 print(n, 4)
16 else:
17 layer_config[n] = {"bits": 8}
18 print(n, 8)
19
20from auto_round import AutoRound
21
22autoround = AutoRound(model, tokenizer, iters=0, layer_config=layer_config)
23autoround.quantize_and_save(format="auto_round", output_dir="tmp_autoround")
24