Views
No views yet
mergekit is a toolkit for merging pre-trained language models. mergekit uses an out-of-core approach to perform unreasonably elaborate merges in resource-constrained situations. Merges can be run entirely on CPU or accelerated with as little as 8 GB of VRAM. Many merging algorithms are supported, with more coming as they catch my attention.
1{
2 "all": {
3 "acc": 0.6152059168567449,
4 "acc_stderr": 0.031951119145286845,
5 "acc_norm": 0.6274010157580394,
6 "acc_norm_stderr": 0.032831804892806175,
7 "mc1": 0.25091799265605874,
8 "mc1_stderr": 0.015176985027707694,
9 "mc2": 0.5006656333594469,
10 "mc2_stderr": 0.01636490303268174
11 },
12 "harness|arc:challenge|25": {
13 "acc": 0.3447098976109215,
14 "acc_stderr": 0.013888816286782112,
15 "acc_norm": 0.3856655290102389,
16 "acc_norm_stderr": 0.01422425097325717
17 },
18 "harness|hellaswag|10": {
19 "acc": 0.34116709818761204,
20 "acc_stderr": 0.004731324409133264,
21 "acc_norm": 0.515435172276439,
22 "acc_norm_stderr": 0.004987403268345035
23 },
24 "harness|hendrycksTest-abstract_algebra|5": {
25 "acc": 0.27,
26 "acc_stderr": 0.04461960433384741,
27 "acc_norm": 0.27,
28 "acc_norm_stderr": 0.04461960433384741
29 },
30 "harness|hendrycksTest-anatomy|5": {
31 "acc": 0.5703703703703704,
32 "acc_stderr": 0.042763494943765995,
33 "acc_norm": 0.5703703703703704,
34 "acc_norm_stderr": 0.042763494943765995
35 },
36 "harness|hendrycksTest-astronomy|5": {
37 "acc": 0.6842105263157895,
38 "acc_stderr": 0.0378272898086547,
39 "acc_norm": 0.6842105263157895,
40 "acc_norm_stderr": 0.0378272898086547
41 },
42 "harness|hendrycksTest-business_ethics|5": {
43 "acc": 0.62,
44 "acc_stderr": 0.048783173121456316,
45 "acc_norm": 0.62,
46 "acc_norm_stderr": 0.048783173121456316
47 },
48 "harness|hendrycksTest-clinical_knowledge|5": {
49 "acc": 0.7169811320754716,
50 "acc_stderr": 0.027724236492700918,
51 "acc_norm": 0.7169811320754716,
52 "acc_norm_stderr": 0.027724236492700918
53 },
54 "harness|hendrycksTest-college_biology|5": {
55 "acc": 0.7638888888888888,
56 "acc_stderr": 0.03551446610810826,
57 "acc_norm": 0.7638888888888888,
58 "acc_norm_stderr": 0.03551446610810826
59 },
60 "harness|hendrycksTest-college_chemistry|5": {
61 "acc": 0.45,
62 "acc_stderr": 0.05,
63 "acc_norm": 0.45,
64 "acc_norm_stderr": 0.05
65 },
66 "harness|hendrycksTest-college_computer_science|5": {
67 "acc": 0.46,
68 "acc_stderr": 0.05009082659620333,
69 "acc_norm": 0.46,
70 "acc_norm_stderr": 0.05009082659620333
71 },
72 "harness|hendrycksTest-college_mathematics|5": {
73 "acc": 0.32,
74 "acc_stderr": 0.04688261722621504,
75 "acc_norm": 0.32,
76 "acc_norm_stderr": 0.04688261722621504
77 },
78 "harness|hendrycksTest-college_medicine|5": {
79 "acc": 0.6358381502890174,
80 "acc_stderr": 0.03669072477416907,
81 "acc_norm": 0.6358381502890174,
82 "acc_norm_stderr": 0.03669072477416907
83 },
84 "harness|hendrycksTest-college_physics|5": {
85 "acc": 0.4019607843137255,
86 "acc_stderr": 0.048786087144669955,
87 "acc_norm": 0.4019607843137255,
88 "acc_norm_stderr": 0.048786087144669955
89 },
90 "harness|hendrycksTest-computer_security|5": {
91 "acc": 0.76,
92 "acc_stderr": 0.04292346959909283,
93 "acc_norm": 0.76,
94 "acc_norm_stderr": 0.04292346959909283
95 },
96 "harness|hendrycksTest-conceptual_physics|5": {
97 "acc": 0.5446808510638298,
98 "acc_stderr": 0.03255525359340355,
99 "acc_norm": 0.5446808510638298,
100 "acc_norm_stderr": 0.03255525359340355
101 },
102 "harness|hendrycksTest-econometrics|5": {
103 "acc": 0.4824561403508772,
104 "acc_stderr": 0.04700708033551038,
105 "acc_norm": 0.4824561403508772,
106 "acc_norm_stderr": 0.04700708033551038
107 },
108 "harness|hendrycksTest-electrical_engineering|5": {
109 "acc": 0.5172413793103449,
110 "acc_stderr": 0.04164188720169375,
111 "acc_norm": 0.5172413793103449,
112 "acc_norm_stderr": 0.04164188720169375
113 },
114 "harness|hendrycksTest-elementary_mathematics|5": {
115 "acc": 0.42857142857142855,
116 "acc_stderr": 0.025487187147859372,
117 "acc_norm": 0.42857142857142855,
118 "acc_norm_stderr": 0.025487187147859372
119 },
120 "harness|hendrycksTest-formal_logic|5": {
121 "acc": 0.3968253968253968,
122 "acc_stderr": 0.043758884927270605,
123 "acc_norm": 0.3968253968253968,
124 "acc_norm_stderr": 0.043758884927270605
125 },
126 "harness|hendrycksTest-global_facts|5": {
127 "acc": 0.34,
128 "acc_stderr": 0.04760952285695236,
129 "acc_norm": 0.34,
130 "acc_norm_stderr": 0.04760952285695236
131 },
132 "harness|hendrycksTest-high_school_biology|5": {
133 "acc": 0.7741935483870968,
134 "acc_stderr": 0.023785577884181015,
135 "acc_norm": 0.7741935483870968,
136 "acc_norm_stderr": 0.023785577884181015
137 },
138 "harness|hendrycksTest-high_school_chemistry|5": {
139 "acc": 0.5123152709359606,
140 "acc_stderr": 0.035169204442208966,
141 "acc_norm": 0.5123152709359606,
142 "acc_norm_stderr": 0.035169204442208966
143 },
144 "harness|hendrycksTest-high_school_computer_science|5": {
145 "acc": 0.66,
146 "acc_stderr": 0.04760952285695237,
147 "acc_norm": 0.66,
148 "acc_norm_stderr": 0.04760952285695237
149 },
150 "harness|hendrycksTest-high_school_european_history|5": {
151 "acc": 0.7636363636363637,
152 "acc_stderr": 0.03317505930009181,
153 "acc_norm": 0.7636363636363637,
154 "acc_norm_stderr": 0.03317505930009181
155 },
156 "harness|hendrycksTest-high_school_geography|5": {
157 "acc": 0.7373737373737373,
158 "acc_stderr": 0.03135305009533085,
159 "acc_norm": 0.7373737373737373,
160 "acc_norm_stderr": 0.03135305009533085
161 },
162 "harness|hendrycksTest-high_school_government_and_politics|5": {
163 "acc": 0.8808290155440415,
164 "acc_stderr": 0.023381935348121437,
165 "acc_norm": 0.8808290155440415,
166 "acc_norm_stderr": 0.023381935348121437
167 },
168 "harness|hendrycksTest-high_school_macroeconomics|5": {
169 "acc": 0.617948717948718,
170 "acc_stderr": 0.024635549163908237,
171 "acc_norm": 0.617948717948718,
172 "acc_norm_stderr": 0.024635549163908237
173 },
174 "harness|hendrycksTest-high_school_mathematics|5": {
175 "acc": 0.2777777777777778,
176 "acc_stderr": 0.027309140588230203,
177 "acc_norm": 0.2777777777777778,
178 "acc_norm_stderr": 0.027309140588230203
179 },
180 "harness|hendrycksTest-high_school_microeconomics|5": {
181 "acc": 0.6512605042016807,
182 "acc_stderr": 0.030956636328566545,
183 "acc_norm": 0.6512605042016807,
184 "acc_norm_stderr": 0.030956636328566545
185 },
186 "harness|hendrycksTest-high_school_physics|5": {
187 "acc": 0.32450331125827814,
188 "acc_stderr": 0.038227469376587525,
189 "acc_norm": 0.32450331125827814,
190 "acc_norm_stderr": 0.038227469376587525
191 },
192 "harness|hendrycksTest-high_school_psychology|5": {
193 "acc": 0.8440366972477065,
194 "acc_stderr": 0.015555802713590158,
195 "acc_norm": 0.8440366972477065,
196 "acc_norm_stderr": 0.015555802713590158
197 },
198 "harness|hendrycksTest-high_school_statistics|5": {
199 "acc": 0.4722222222222222,
200 "acc_stderr": 0.0340470532865388,
201 "acc_norm": 0.4722222222222222,
202 "acc_norm_stderr": 0.0340470532865388
203 },
204 "harness|hendrycksTest-high_school_us_history|5": {
205 "acc": 0.8431372549019608,
206 "acc_stderr": 0.025524722324553346,
207 "acc_norm": 0.8431372549019608,
208 "acc_norm_stderr": 0.025524722324553346
209 },
210 "harness|hendrycksTest-high_school_world_history|5": {
211 "acc": 0.810126582278481,
212 "acc_stderr": 0.025530100460233497,
213 "acc_norm": 0.810126582278481,
214 "acc_norm_stderr": 0.025530100460233497
215 },
216 "harness|hendrycksTest-human_aging|5": {
217 "acc": 0.7174887892376681,
218 "acc_stderr": 0.03021683101150877,
219 "acc_norm": 0.7174887892376681,
220 "acc_norm_stderr": 0.03021683101150877
221 },
222 "harness|hendrycksTest-human_sexuality|5": {
223 "acc": 0.7786259541984732,
224 "acc_stderr": 0.0364129708131373,
225 "acc_norm": 0.7786259541984732,
226 "acc_norm_stderr": 0.0364129708131373
227 },
228 "harness|hendrycksTest-international_law|5": {
229 "acc": 0.7768595041322314,
230 "acc_stderr": 0.03800754475228733,
231 "acc_norm": 0.7768595041322314,
232 "acc_norm_stderr": 0.03800754475228733
233 },
234 "harness|hendrycksTest-jurisprudence|5": {
235 "acc": 0.8148148148148148,
236 "acc_stderr": 0.03755265865037181,
237 "acc_norm": 0.8148148148148148,
238 "acc_norm_stderr": 0.03755265865037181
239 },
240 "harness|hendrycksTest-logical_fallacies|5": {
241 "acc": 0.7914110429447853,
242 "acc_stderr": 0.031921934489347235,
243 "acc_norm": 0.7914110429447853,
244 "acc_norm_stderr": 0.031921934489347235
245 },
246 "harness|hendrycksTest-machine_learning|5": {
247 "acc": 0.5446428571428571,
248 "acc_stderr": 0.04726835553719097,
249 "acc_norm": 0.5446428571428571,
250 "acc_norm_stderr": 0.04726835553719097
251 },
252 "harness|hendrycksTest-management|5": {
253 "acc": 0.8349514563106796,
254 "acc_stderr": 0.036756688322331886,
255 "acc_norm": 0.8349514563106796,
256 "acc_norm_stderr": 0.036756688322331886
257 },
258 "harness|hendrycksTest-marketing|5": {
259 "acc": 0.8290598290598291,
260 "acc_stderr": 0.024662496845209804,
261 "acc_norm": 0.8290598290598291,
262 "acc_norm_stderr": 0.024662496845209804
263 },
264 "harness|hendrycksTest-medical_genetics|5": {
265 "acc": 0.69,
266 "acc_stderr": 0.04648231987117316,
267 "acc_norm": 0.69,
268 "acc_norm_stderr": 0.04648231987117316
269 },
270 "harness|hendrycksTest-miscellaneous|5": {
271 "acc": 0.8250319284802043,
272 "acc_stderr": 0.013586619219903324,
273 "acc_norm": 0.8250319284802043,
274 "acc_norm_stderr": 0.013586619219903324
275 },
276 "harness|hendrycksTest-moral_disputes|5": {
277 "acc": 0.7283236994219653,
278 "acc_stderr": 0.023948512905468348,
279 "acc_norm": 0.7283236994219653,
280 "acc_norm_stderr": 0.023948512905468348
281 },
282 "harness|hendrycksTest-moral_scenarios|5": {
283 "acc": 0.36312849162011174,
284 "acc_stderr": 0.016083749986853704,
285 "acc_norm": 0.36312849162011174,
286 "acc_norm_stderr": 0.016083749986853704
287 },
288 "harness|hendrycksTest-nutrition|5": {
289 "acc": 0.7450980392156863,
290 "acc_stderr": 0.02495418432487991,
291 "acc_norm": 0.7450980392156863,
292 "acc_norm_stderr": 0.02495418432487991
293 },
294 "harness|hendrycksTest-philosophy|5": {
295 "acc": 0.7202572347266881,
296 "acc_stderr": 0.02549425935069491,
297 "acc_norm": 0.7202572347266881,
298 "acc_norm_stderr": 0.02549425935069491
299 },
300 "harness|hendrycksTest-prehistory|5": {
301 "acc": 0.7530864197530864,
302 "acc_stderr": 0.023993501709042114,
303 "acc_norm": 0.7530864197530864,
304 "acc_norm_stderr": 0.023993501709042114
305 },
306 "harness|hendrycksTest-professional_accounting|5": {
307 "acc": 0.4787234042553192,
308 "acc_stderr": 0.029800481645628693,
309 "acc_norm": 0.4787234042553192,
310 "acc_norm_stderr": 0.029800481645628693
311 },
312 "harness|hendrycksTest-professional_law|5": {
313 "acc": 0.4367666232073012,
314 "acc_stderr": 0.01266770191960366,
315 "acc_norm": 0.4367666232073012,
316 "acc_norm_stderr": 0.01266770191960366
317 },
318 "harness|hendrycksTest-professional_medicine|5": {
319 "acc": 0.6176470588235294,
320 "acc_stderr": 0.029520095697687765,
321 "acc_norm": 0.6176470588235294,
322 "acc_norm_stderr": 0.029520095697687765
323 },
324 "harness|hendrycksTest-professional_psychology|5": {
325 "acc": 0.6699346405228758,
326 "acc_stderr": 0.019023726160724553,
327 "acc_norm": 0.6699346405228758,
328 "acc_norm_stderr": 0.019023726160724553
329 },
330 "harness|hendrycksTest-public_relations|5": {
331 "acc": 0.6545454545454545,
332 "acc_stderr": 0.04554619617541054,
333 "acc_norm": 0.6545454545454545,
334 "acc_norm_stderr": 0.04554619617541054
335 },
336 "harness|hendrycksTest-security_studies|5": {
337 "acc": 0.726530612244898,
338 "acc_stderr": 0.028535560337128445,
339 "acc_norm": 0.726530612244898,
340 "acc_norm_stderr": 0.028535560337128445
341 },
342 "harness|hendrycksTest-sociology|5": {
343 "acc": 0.845771144278607,
344 "acc_stderr": 0.025538433368578334,
345 "acc_norm": 0.845771144278607,
346 "acc_norm_stderr": 0.025538433368578334
347 },
348 "harness|hendrycksTest-us_foreign_policy|5": {
349 "acc": 0.85,
350 "acc_stderr": 0.03588702812826371,
351 "acc_norm": 0.85,
352 "acc_norm_stderr": 0.03588702812826371
353 },
354 "harness|hendrycksTest-virology|5": {
355 "acc": 0.5542168674698795,
356 "acc_stderr": 0.03869543323472101,
357 "acc_norm": 0.5542168674698795,
358 "acc_norm_stderr": 0.03869543323472101
359 },
360 "harness|hendrycksTest-world_religions|5": {
361 "acc": 0.8362573099415205,
362 "acc_stderr": 0.028380919596145866,
363 "acc_norm": 0.8362573099415205,
364 "acc_norm_stderr": 0.028380919596145866
365 },
366 "harness|truthfulqa:mc|0": {
367 "mc1": 0.25091799265605874,
368 "mc1_stderr": 0.015176985027707694,
369 "mc2": 0.5006656333594469,
370 "mc2_stderr": 0.01636490303268174
371 },
372 "harness|winogrande|5": {
373 "acc": 0.7261247040252565,
374 "acc_stderr": 0.012533292732620296
375 },
376 "harness|gsm8k|5": {
377 "acc": 0.0,
378 "acc_stderr": 0.0
379 }
380}
381| Metric | Value |
|---|---|
| Avg. | 46.02 |
| AI2 Reasoning Challenge (25-Shot) | 38.57 |
| HellaSwag (10-Shot) | 51.54 |
| MMLU (5-Shot) | 63.36 |
| TruthfulQA (0-shot) | 50.07 |
| Winogrande (5-shot) | 72.61 |
| GSM8k (5-shot) | 0.00 |