Views
No views yet
mergekit is a toolkit for merging pre-trained language models. mergekit uses an out-of-core approach to perform unreasonably elaborate merges in resource-constrained situations. Merges can be run entirely on CPU or accelerated with as little as 8 GB of VRAM. Many merging algorithms are supported, with more coming as they catch my attention.
1{
2 "all": {
3 "acc": 0.6547370286177235,
4 "acc_stderr": 0.03204709242170183,
5 "acc_norm": 0.6537337854798912,
6 "acc_norm_stderr": 0.03272317883588649,
7 "mc1": 0.5189718482252142,
8 "mc1_stderr": 0.01749089640576236,
9 "mc2": 0.6631825155680797,
10 "mc2_stderr": 0.01527641053841743
11 },
12 "harness|arc:challenge|25": {
13 "acc": 0.6953924914675768,
14 "acc_stderr": 0.013449522109932485,
15 "acc_norm": 0.7175767918088737,
16 "acc_norm_stderr": 0.013155456884097225
17 },
18 "harness|hellaswag|10": {
19 "acc": 0.7120095598486357,
20 "acc_stderr": 0.004519011688417168,
21 "acc_norm": 0.8820952001593309,
22 "acc_norm_stderr": 0.003218362717491129
23 },
24 "harness|hendrycksTest-abstract_algebra|5": {
25 "acc": 0.33,
26 "acc_stderr": 0.047258156262526045,
27 "acc_norm": 0.33,
28 "acc_norm_stderr": 0.047258156262526045
29 },
30 "harness|hendrycksTest-anatomy|5": {
31 "acc": 0.6296296296296297,
32 "acc_stderr": 0.041716541613545426,
33 "acc_norm": 0.6296296296296297,
34 "acc_norm_stderr": 0.041716541613545426
35 },
36 "harness|hendrycksTest-astronomy|5": {
37 "acc": 0.7105263157894737,
38 "acc_stderr": 0.03690677986137283,
39 "acc_norm": 0.7105263157894737,
40 "acc_norm_stderr": 0.03690677986137283
41 },
42 "harness|hendrycksTest-business_ethics|5": {
43 "acc": 0.64,
44 "acc_stderr": 0.04824181513244218,
45 "acc_norm": 0.64,
46 "acc_norm_stderr": 0.04824181513244218
47 },
48 "harness|hendrycksTest-clinical_knowledge|5": {
49 "acc": 0.7056603773584905,
50 "acc_stderr": 0.02804918631569525,
51 "acc_norm": 0.7056603773584905,
52 "acc_norm_stderr": 0.02804918631569525
53 },
54 "harness|hendrycksTest-college_biology|5": {
55 "acc": 0.7638888888888888,
56 "acc_stderr": 0.03551446610810826,
57 "acc_norm": 0.7638888888888888,
58 "acc_norm_stderr": 0.03551446610810826
59 },
60 "harness|hendrycksTest-college_chemistry|5": {
61 "acc": 0.48,
62 "acc_stderr": 0.050211673156867795,
63 "acc_norm": 0.48,
64 "acc_norm_stderr": 0.050211673156867795
65 },
66 "harness|hendrycksTest-college_computer_science|5": {
67 "acc": 0.54,
68 "acc_stderr": 0.05009082659620333,
69 "acc_norm": 0.54,
70 "acc_norm_stderr": 0.05009082659620333
71 },
72 "harness|hendrycksTest-college_mathematics|5": {
73 "acc": 0.31,
74 "acc_stderr": 0.04648231987117316,
75 "acc_norm": 0.31,
76 "acc_norm_stderr": 0.04648231987117316
77 },
78 "harness|hendrycksTest-college_medicine|5": {
79 "acc": 0.6705202312138728,
80 "acc_stderr": 0.03583901754736411,
81 "acc_norm": 0.6705202312138728,
82 "acc_norm_stderr": 0.03583901754736411
83 },
84 "harness|hendrycksTest-college_physics|5": {
85 "acc": 0.4215686274509804,
86 "acc_stderr": 0.04913595201274498,
87 "acc_norm": 0.4215686274509804,
88 "acc_norm_stderr": 0.04913595201274498
89 },
90 "harness|hendrycksTest-computer_security|5": {
91 "acc": 0.78,
92 "acc_stderr": 0.04163331998932263,
93 "acc_norm": 0.78,
94 "acc_norm_stderr": 0.04163331998932263
95 },
96 "harness|hendrycksTest-conceptual_physics|5": {
97 "acc": 0.5787234042553191,
98 "acc_stderr": 0.03227834510146268,
99 "acc_norm": 0.5787234042553191,
100 "acc_norm_stderr": 0.03227834510146268
101 },
102 "harness|hendrycksTest-econometrics|5": {
103 "acc": 0.5,
104 "acc_stderr": 0.047036043419179864,
105 "acc_norm": 0.5,
106 "acc_norm_stderr": 0.047036043419179864
107 },
108 "harness|hendrycksTest-electrical_engineering|5": {
109 "acc": 0.5586206896551724,
110 "acc_stderr": 0.04137931034482758,
111 "acc_norm": 0.5586206896551724,
112 "acc_norm_stderr": 0.04137931034482758
113 },
114 "harness|hendrycksTest-elementary_mathematics|5": {
115 "acc": 0.42857142857142855,
116 "acc_stderr": 0.02548718714785938,
117 "acc_norm": 0.42857142857142855,
118 "acc_norm_stderr": 0.02548718714785938
119 },
120 "harness|hendrycksTest-formal_logic|5": {
121 "acc": 0.47619047619047616,
122 "acc_stderr": 0.04467062628403273,
123 "acc_norm": 0.47619047619047616,
124 "acc_norm_stderr": 0.04467062628403273
125 },
126 "harness|hendrycksTest-global_facts|5": {
127 "acc": 0.33,
128 "acc_stderr": 0.04725815626252604,
129 "acc_norm": 0.33,
130 "acc_norm_stderr": 0.04725815626252604
131 },
132 "harness|hendrycksTest-high_school_biology|5": {
133 "acc": 0.7903225806451613,
134 "acc_stderr": 0.023157879349083525,
135 "acc_norm": 0.7903225806451613,
136 "acc_norm_stderr": 0.023157879349083525
137 },
138 "harness|hendrycksTest-high_school_chemistry|5": {
139 "acc": 0.4876847290640394,
140 "acc_stderr": 0.035169204442208966,
141 "acc_norm": 0.4876847290640394,
142 "acc_norm_stderr": 0.035169204442208966
143 },
144 "harness|hendrycksTest-high_school_computer_science|5": {
145 "acc": 0.68,
146 "acc_stderr": 0.04688261722621505,
147 "acc_norm": 0.68,
148 "acc_norm_stderr": 0.04688261722621505
149 },
150 "harness|hendrycksTest-high_school_european_history|5": {
151 "acc": 0.7878787878787878,
152 "acc_stderr": 0.03192271569548301,
153 "acc_norm": 0.7878787878787878,
154 "acc_norm_stderr": 0.03192271569548301
155 },
156 "harness|hendrycksTest-high_school_geography|5": {
157 "acc": 0.797979797979798,
158 "acc_stderr": 0.02860620428922987,
159 "acc_norm": 0.797979797979798,
160 "acc_norm_stderr": 0.02860620428922987
161 },
162 "harness|hendrycksTest-high_school_government_and_politics|5": {
163 "acc": 0.9015544041450777,
164 "acc_stderr": 0.021500249576033456,
165 "acc_norm": 0.9015544041450777,
166 "acc_norm_stderr": 0.021500249576033456
167 },
168 "harness|hendrycksTest-high_school_macroeconomics|5": {
169 "acc": 0.6666666666666666,
170 "acc_stderr": 0.023901157979402538,
171 "acc_norm": 0.6666666666666666,
172 "acc_norm_stderr": 0.023901157979402538
173 },
174 "harness|hendrycksTest-high_school_mathematics|5": {
175 "acc": 0.35185185185185186,
176 "acc_stderr": 0.029116617606083008,
177 "acc_norm": 0.35185185185185186,
178 "acc_norm_stderr": 0.029116617606083008
179 },
180 "harness|hendrycksTest-high_school_microeconomics|5": {
181 "acc": 0.6722689075630253,
182 "acc_stderr": 0.03048991141767323,
183 "acc_norm": 0.6722689075630253,
184 "acc_norm_stderr": 0.03048991141767323
185 },
186 "harness|hendrycksTest-high_school_physics|5": {
187 "acc": 0.36423841059602646,
188 "acc_stderr": 0.03929111781242742,
189 "acc_norm": 0.36423841059602646,
190 "acc_norm_stderr": 0.03929111781242742
191 },
192 "harness|hendrycksTest-high_school_psychology|5": {
193 "acc": 0.8440366972477065,
194 "acc_stderr": 0.015555802713590167,
195 "acc_norm": 0.8440366972477065,
196 "acc_norm_stderr": 0.015555802713590167
197 },
198 "harness|hendrycksTest-high_school_statistics|5": {
199 "acc": 0.5092592592592593,
200 "acc_stderr": 0.034093869469927006,
201 "acc_norm": 0.5092592592592593,
202 "acc_norm_stderr": 0.034093869469927006
203 },
204 "harness|hendrycksTest-high_school_us_history|5": {
205 "acc": 0.8333333333333334,
206 "acc_stderr": 0.026156867523931045,
207 "acc_norm": 0.8333333333333334,
208 "acc_norm_stderr": 0.026156867523931045
209 },
210 "harness|hendrycksTest-high_school_world_history|5": {
211 "acc": 0.7848101265822784,
212 "acc_stderr": 0.02675082699467618,
213 "acc_norm": 0.7848101265822784,
214 "acc_norm_stderr": 0.02675082699467618
215 },
216 "harness|hendrycksTest-human_aging|5": {
217 "acc": 0.6905829596412556,
218 "acc_stderr": 0.03102441174057221,
219 "acc_norm": 0.6905829596412556,
220 "acc_norm_stderr": 0.03102441174057221
221 },
222 "harness|hendrycksTest-human_sexuality|5": {
223 "acc": 0.7786259541984732,
224 "acc_stderr": 0.03641297081313729,
225 "acc_norm": 0.7786259541984732,
226 "acc_norm_stderr": 0.03641297081313729
227 },
228 "harness|hendrycksTest-international_law|5": {
229 "acc": 0.7933884297520661,
230 "acc_stderr": 0.03695980128098824,
231 "acc_norm": 0.7933884297520661,
232 "acc_norm_stderr": 0.03695980128098824
233 },
234 "harness|hendrycksTest-jurisprudence|5": {
235 "acc": 0.7870370370370371,
236 "acc_stderr": 0.0395783547198098,
237 "acc_norm": 0.7870370370370371,
238 "acc_norm_stderr": 0.0395783547198098
239 },
240 "harness|hendrycksTest-logical_fallacies|5": {
241 "acc": 0.7730061349693251,
242 "acc_stderr": 0.03291099578615769,
243 "acc_norm": 0.7730061349693251,
244 "acc_norm_stderr": 0.03291099578615769
245 },
246 "harness|hendrycksTest-machine_learning|5": {
247 "acc": 0.45535714285714285,
248 "acc_stderr": 0.047268355537191,
249 "acc_norm": 0.45535714285714285,
250 "acc_norm_stderr": 0.047268355537191
251 },
252 "harness|hendrycksTest-management|5": {
253 "acc": 0.7766990291262136,
254 "acc_stderr": 0.04123553189891431,
255 "acc_norm": 0.7766990291262136,
256 "acc_norm_stderr": 0.04123553189891431
257 },
258 "harness|hendrycksTest-marketing|5": {
259 "acc": 0.8760683760683761,
260 "acc_stderr": 0.021586494001281376,
261 "acc_norm": 0.8760683760683761,
262 "acc_norm_stderr": 0.021586494001281376
263 },
264 "harness|hendrycksTest-medical_genetics|5": {
265 "acc": 0.72,
266 "acc_stderr": 0.045126085985421276,
267 "acc_norm": 0.72,
268 "acc_norm_stderr": 0.045126085985421276
269 },
270 "harness|hendrycksTest-miscellaneous|5": {
271 "acc": 0.8275862068965517,
272 "acc_stderr": 0.013507943909371798,
273 "acc_norm": 0.8275862068965517,
274 "acc_norm_stderr": 0.013507943909371798
275 },
276 "harness|hendrycksTest-moral_disputes|5": {
277 "acc": 0.7427745664739884,
278 "acc_stderr": 0.02353292543104429,
279 "acc_norm": 0.7427745664739884,
280 "acc_norm_stderr": 0.02353292543104429
281 },
282 "harness|hendrycksTest-moral_scenarios|5": {
283 "acc": 0.4312849162011173,
284 "acc_stderr": 0.016563829399047707,
285 "acc_norm": 0.4312849162011173,
286 "acc_norm_stderr": 0.016563829399047707
287 },
288 "harness|hendrycksTest-nutrition|5": {
289 "acc": 0.7320261437908496,
290 "acc_stderr": 0.025360603796242557,
291 "acc_norm": 0.7320261437908496,
292 "acc_norm_stderr": 0.025360603796242557
293 },
294 "harness|hendrycksTest-philosophy|5": {
295 "acc": 0.7170418006430869,
296 "acc_stderr": 0.02558306248998481,
297 "acc_norm": 0.7170418006430869,
298 "acc_norm_stderr": 0.02558306248998481
299 },
300 "harness|hendrycksTest-prehistory|5": {
301 "acc": 0.7438271604938271,
302 "acc_stderr": 0.024288533637726095,
303 "acc_norm": 0.7438271604938271,
304 "acc_norm_stderr": 0.024288533637726095
305 },
306 "harness|hendrycksTest-professional_accounting|5": {
307 "acc": 0.46808510638297873,
308 "acc_stderr": 0.029766675075873866,
309 "acc_norm": 0.46808510638297873,
310 "acc_norm_stderr": 0.029766675075873866
311 },
312 "harness|hendrycksTest-professional_law|5": {
313 "acc": 0.4726205997392438,
314 "acc_stderr": 0.012751075788015055,
315 "acc_norm": 0.4726205997392438,
316 "acc_norm_stderr": 0.012751075788015055
317 },
318 "harness|hendrycksTest-professional_medicine|5": {
319 "acc": 0.6801470588235294,
320 "acc_stderr": 0.02833295951403121,
321 "acc_norm": 0.6801470588235294,
322 "acc_norm_stderr": 0.02833295951403121
323 },
324 "harness|hendrycksTest-professional_psychology|5": {
325 "acc": 0.6748366013071896,
326 "acc_stderr": 0.018950886770806315,
327 "acc_norm": 0.6748366013071896,
328 "acc_norm_stderr": 0.018950886770806315
329 },
330 "harness|hendrycksTest-public_relations|5": {
331 "acc": 0.6909090909090909,
332 "acc_stderr": 0.044262946482000985,
333 "acc_norm": 0.6909090909090909,
334 "acc_norm_stderr": 0.044262946482000985
335 },
336 "harness|hendrycksTest-security_studies|5": {
337 "acc": 0.7306122448979592,
338 "acc_stderr": 0.02840125202902294,
339 "acc_norm": 0.7306122448979592,
340 "acc_norm_stderr": 0.02840125202902294
341 },
342 "harness|hendrycksTest-sociology|5": {
343 "acc": 0.835820895522388,
344 "acc_stderr": 0.026193923544454115,
345 "acc_norm": 0.835820895522388,
346 "acc_norm_stderr": 0.026193923544454115
347 },
348 "harness|hendrycksTest-us_foreign_policy|5": {
349 "acc": 0.85,
350 "acc_stderr": 0.03588702812826371,
351 "acc_norm": 0.85,
352 "acc_norm_stderr": 0.03588702812826371
353 },
354 "harness|hendrycksTest-virology|5": {
355 "acc": 0.5602409638554217,
356 "acc_stderr": 0.03864139923699122,
357 "acc_norm": 0.5602409638554217,
358 "acc_norm_stderr": 0.03864139923699122
359 },
360 "harness|hendrycksTest-world_religions|5": {
361 "acc": 0.8362573099415205,
362 "acc_stderr": 0.028380919596145866,
363 "acc_norm": 0.8362573099415205,
364 "acc_norm_stderr": 0.028380919596145866
365 },
366 "harness|truthfulqa:mc|0": {
367 "mc1": 0.5189718482252142,
368 "mc1_stderr": 0.01749089640576236,
369 "mc2": 0.6631825155680797,
370 "mc2_stderr": 0.01527641053841743
371 },
372 "harness|winogrande|5": {
373 "acc": 0.8437253354380426,
374 "acc_stderr": 0.01020535179187352
375 },
376 "harness|gsm8k|5": {
377 "acc": 0.7172100075815011,
378 "acc_stderr": 0.012405020417873619
379 }
380}
381| Metric | Value |
|---|---|
| Avg. | 74.54 |
| AI2 Reasoning Challenge (25-Shot) | 71.76 |
| HellaSwag (10-Shot) | 88.21 |
| MMLU (5-Shot) | 64.86 |
| TruthfulQA (0-shot) | 66.32 |
| Winogrande (5-shot) | 84.37 |
| GSM8k (5-shot) | 71.72 |