Views
No views yet
1slices:
2 - sources:
3 - model: Weyaxi/Einstein-v5-v0.2-7B
4 layer_range: [0, 32]
5 - model: argilla/CapybaraHermes-2.5-Mistral-7B
6 layer_range: [0, 32]
7merge_method: slerp
8base_model: argilla/CapybaraHermes-2.5-Mistral-7B
9parameters:
10 t:
11 - filter: self_attn
12 value: [0, 0.5, 0.3, 0.7, 1]
13 - filter: mlp
14 value: [1, 0.5, 0.7, 0.3, 0]
15 - value: 0.5
16dtype: bfloat161{
2 "all": {
3 "acc": 0.6312165296664113,
4 "acc_stderr": 0.03236370559394293,
5 "acc_norm": 0.6324439925872714,
6 "acc_norm_stderr": 0.033019786616359854,
7 "mc1": 0.39657282741738065,
8 "mc1_stderr": 0.017124930942023518,
9 "mc2": 0.5688038233837539,
10 "mc2_stderr": 0.015263125204118244
11 },
12 "harness|arc:challenge|25": {
13 "acc": 0.6126279863481229,
14 "acc_stderr": 0.014235872487909869,
15 "acc_norm": 0.6348122866894198,
16 "acc_norm_stderr": 0.014070265519268802
17 },
18 "harness|hellaswag|10": {
19 "acc": 0.643397729535949,
20 "acc_stderr": 0.00478016987333285,
21 "acc_norm": 0.8378809002190799,
22 "acc_norm_stderr": 0.0036780679944244735
23 },
24 "harness|hendrycksTest-abstract_algebra|5": {
25 "acc": 0.32,
26 "acc_stderr": 0.046882617226215034,
27 "acc_norm": 0.32,
28 "acc_norm_stderr": 0.046882617226215034
29 },
30 "harness|hendrycksTest-anatomy|5": {
31 "acc": 0.6074074074074074,
32 "acc_stderr": 0.0421850621536888,
33 "acc_norm": 0.6074074074074074,
34 "acc_norm_stderr": 0.0421850621536888
35 },
36 "harness|hendrycksTest-astronomy|5": {
37 "acc": 0.6907894736842105,
38 "acc_stderr": 0.037610708698674805,
39 "acc_norm": 0.6907894736842105,
40 "acc_norm_stderr": 0.037610708698674805
41 },
42 "harness|hendrycksTest-business_ethics|5": {
43 "acc": 0.61,
44 "acc_stderr": 0.04902071300001975,
45 "acc_norm": 0.61,
46 "acc_norm_stderr": 0.04902071300001975
47 },
48 "harness|hendrycksTest-clinical_knowledge|5": {
49 "acc": 0.6754716981132075,
50 "acc_stderr": 0.02881561571343211,
51 "acc_norm": 0.6754716981132075,
52 "acc_norm_stderr": 0.02881561571343211
53 },
54 "harness|hendrycksTest-college_biology|5": {
55 "acc": 0.7291666666666666,
56 "acc_stderr": 0.03716177437566017,
57 "acc_norm": 0.7291666666666666,
58 "acc_norm_stderr": 0.03716177437566017
59 },
60 "harness|hendrycksTest-college_chemistry|5": {
61 "acc": 0.47,
62 "acc_stderr": 0.05016135580465919,
63 "acc_norm": 0.47,
64 "acc_norm_stderr": 0.05016135580465919
65 },
66 "harness|hendrycksTest-college_computer_science|5": {
67 "acc": 0.48,
68 "acc_stderr": 0.050211673156867795,
69 "acc_norm": 0.48,
70 "acc_norm_stderr": 0.050211673156867795
71 },
72 "harness|hendrycksTest-college_mathematics|5": {
73 "acc": 0.34,
74 "acc_stderr": 0.04760952285695235,
75 "acc_norm": 0.34,
76 "acc_norm_stderr": 0.04760952285695235
77 },
78 "harness|hendrycksTest-college_medicine|5": {
79 "acc": 0.6011560693641619,
80 "acc_stderr": 0.037336266553835096,
81 "acc_norm": 0.6011560693641619,
82 "acc_norm_stderr": 0.037336266553835096
83 },
84 "harness|hendrycksTest-college_physics|5": {
85 "acc": 0.29411764705882354,
86 "acc_stderr": 0.04533838195929775,
87 "acc_norm": 0.29411764705882354,
88 "acc_norm_stderr": 0.04533838195929775
89 },
90 "harness|hendrycksTest-computer_security|5": {
91 "acc": 0.72,
92 "acc_stderr": 0.045126085985421276,
93 "acc_norm": 0.72,
94 "acc_norm_stderr": 0.045126085985421276
95 },
96 "harness|hendrycksTest-conceptual_physics|5": {
97 "acc": 0.5659574468085107,
98 "acc_stderr": 0.03240038086792747,
99 "acc_norm": 0.5659574468085107,
100 "acc_norm_stderr": 0.03240038086792747
101 },
102 "harness|hendrycksTest-econometrics|5": {
103 "acc": 0.5,
104 "acc_stderr": 0.047036043419179864,
105 "acc_norm": 0.5,
106 "acc_norm_stderr": 0.047036043419179864
107 },
108 "harness|hendrycksTest-electrical_engineering|5": {
109 "acc": 0.5448275862068965,
110 "acc_stderr": 0.04149886942192117,
111 "acc_norm": 0.5448275862068965,
112 "acc_norm_stderr": 0.04149886942192117
113 },
114 "harness|hendrycksTest-elementary_mathematics|5": {
115 "acc": 0.4021164021164021,
116 "acc_stderr": 0.02525303255499769,
117 "acc_norm": 0.4021164021164021,
118 "acc_norm_stderr": 0.02525303255499769
119 },
120 "harness|hendrycksTest-formal_logic|5": {
121 "acc": 0.42063492063492064,
122 "acc_stderr": 0.04415438226743744,
123 "acc_norm": 0.42063492063492064,
124 "acc_norm_stderr": 0.04415438226743744
125 },
126 "harness|hendrycksTest-global_facts|5": {
127 "acc": 0.39,
128 "acc_stderr": 0.04902071300001975,
129 "acc_norm": 0.39,
130 "acc_norm_stderr": 0.04902071300001975
131 },
132 "harness|hendrycksTest-high_school_biology|5": {
133 "acc": 0.7774193548387097,
134 "acc_stderr": 0.02366421667164251,
135 "acc_norm": 0.7774193548387097,
136 "acc_norm_stderr": 0.02366421667164251
137 },
138 "harness|hendrycksTest-high_school_chemistry|5": {
139 "acc": 0.4876847290640394,
140 "acc_stderr": 0.035169204442208966,
141 "acc_norm": 0.4876847290640394,
142 "acc_norm_stderr": 0.035169204442208966
143 },
144 "harness|hendrycksTest-high_school_computer_science|5": {
145 "acc": 0.68,
146 "acc_stderr": 0.04688261722621505,
147 "acc_norm": 0.68,
148 "acc_norm_stderr": 0.04688261722621505
149 },
150 "harness|hendrycksTest-high_school_european_history|5": {
151 "acc": 0.7818181818181819,
152 "acc_stderr": 0.03225078108306289,
153 "acc_norm": 0.7818181818181819,
154 "acc_norm_stderr": 0.03225078108306289
155 },
156 "harness|hendrycksTest-high_school_geography|5": {
157 "acc": 0.803030303030303,
158 "acc_stderr": 0.02833560973246336,
159 "acc_norm": 0.803030303030303,
160 "acc_norm_stderr": 0.02833560973246336
161 },
162 "harness|hendrycksTest-high_school_government_and_politics|5": {
163 "acc": 0.8549222797927462,
164 "acc_stderr": 0.025416343096306433,
165 "acc_norm": 0.8549222797927462,
166 "acc_norm_stderr": 0.025416343096306433
167 },
168 "harness|hendrycksTest-high_school_macroeconomics|5": {
169 "acc": 0.6435897435897436,
170 "acc_stderr": 0.02428314052946731,
171 "acc_norm": 0.6435897435897436,
172 "acc_norm_stderr": 0.02428314052946731
173 },
174 "harness|hendrycksTest-high_school_mathematics|5": {
175 "acc": 0.32592592592592595,
176 "acc_stderr": 0.028578348365473072,
177 "acc_norm": 0.32592592592592595,
178 "acc_norm_stderr": 0.028578348365473072
179 },
180 "harness|hendrycksTest-high_school_microeconomics|5": {
181 "acc": 0.6638655462184874,
182 "acc_stderr": 0.030684737115135367,
183 "acc_norm": 0.6638655462184874,
184 "acc_norm_stderr": 0.030684737115135367
185 },
186 "harness|hendrycksTest-high_school_physics|5": {
187 "acc": 0.31788079470198677,
188 "acc_stderr": 0.038020397601079024,
189 "acc_norm": 0.31788079470198677,
190 "acc_norm_stderr": 0.038020397601079024
191 },
192 "harness|hendrycksTest-high_school_psychology|5": {
193 "acc": 0.8220183486238533,
194 "acc_stderr": 0.01639943636661289,
195 "acc_norm": 0.8220183486238533,
196 "acc_norm_stderr": 0.01639943636661289
197 },
198 "harness|hendrycksTest-high_school_statistics|5": {
199 "acc": 0.5185185185185185,
200 "acc_stderr": 0.034076320938540516,
201 "acc_norm": 0.5185185185185185,
202 "acc_norm_stderr": 0.034076320938540516
203 },
204 "harness|hendrycksTest-high_school_us_history|5": {
205 "acc": 0.803921568627451,
206 "acc_stderr": 0.027865942286639318,
207 "acc_norm": 0.803921568627451,
208 "acc_norm_stderr": 0.027865942286639318
209 },
210 "harness|hendrycksTest-high_school_world_history|5": {
211 "acc": 0.7974683544303798,
212 "acc_stderr": 0.026160568246601453,
213 "acc_norm": 0.7974683544303798,
214 "acc_norm_stderr": 0.026160568246601453
215 },
216 "harness|hendrycksTest-human_aging|5": {
217 "acc": 0.6995515695067265,
218 "acc_stderr": 0.03076935200822914,
219 "acc_norm": 0.6995515695067265,
220 "acc_norm_stderr": 0.03076935200822914
221 },
222 "harness|hendrycksTest-human_sexuality|5": {
223 "acc": 0.7480916030534351,
224 "acc_stderr": 0.03807387116306085,
225 "acc_norm": 0.7480916030534351,
226 "acc_norm_stderr": 0.03807387116306085
227 },
228 "harness|hendrycksTest-international_law|5": {
229 "acc": 0.8016528925619835,
230 "acc_stderr": 0.036401182719909456,
231 "acc_norm": 0.8016528925619835,
232 "acc_norm_stderr": 0.036401182719909456
233 },
234 "harness|hendrycksTest-jurisprudence|5": {
235 "acc": 0.8055555555555556,
236 "acc_stderr": 0.038260763248848646,
237 "acc_norm": 0.8055555555555556,
238 "acc_norm_stderr": 0.038260763248848646
239 },
240 "harness|hendrycksTest-logical_fallacies|5": {
241 "acc": 0.754601226993865,
242 "acc_stderr": 0.03380939813943354,
243 "acc_norm": 0.754601226993865,
244 "acc_norm_stderr": 0.03380939813943354
245 },
246 "harness|hendrycksTest-machine_learning|5": {
247 "acc": 0.44642857142857145,
248 "acc_stderr": 0.04718471485219588,
249 "acc_norm": 0.44642857142857145,
250 "acc_norm_stderr": 0.04718471485219588
251 },
252 "harness|hendrycksTest-management|5": {
253 "acc": 0.7961165048543689,
254 "acc_stderr": 0.039891398595317706,
255 "acc_norm": 0.7961165048543689,
256 "acc_norm_stderr": 0.039891398595317706
257 },
258 "harness|hendrycksTest-marketing|5": {
259 "acc": 0.8589743589743589,
260 "acc_stderr": 0.02280138253459754,
261 "acc_norm": 0.8589743589743589,
262 "acc_norm_stderr": 0.02280138253459754
263 },
264 "harness|hendrycksTest-medical_genetics|5": {
265 "acc": 0.73,
266 "acc_stderr": 0.044619604333847394,
267 "acc_norm": 0.73,
268 "acc_norm_stderr": 0.044619604333847394
269 },
270 "harness|hendrycksTest-miscellaneous|5": {
271 "acc": 0.8084291187739464,
272 "acc_stderr": 0.014072859310451949,
273 "acc_norm": 0.8084291187739464,
274 "acc_norm_stderr": 0.014072859310451949
275 },
276 "harness|hendrycksTest-moral_disputes|5": {
277 "acc": 0.7312138728323699,
278 "acc_stderr": 0.023868003262500104,
279 "acc_norm": 0.7312138728323699,
280 "acc_norm_stderr": 0.023868003262500104
281 },
282 "harness|hendrycksTest-moral_scenarios|5": {
283 "acc": 0.24916201117318434,
284 "acc_stderr": 0.014465893829859924,
285 "acc_norm": 0.24916201117318434,
286 "acc_norm_stderr": 0.014465893829859924
287 },
288 "harness|hendrycksTest-nutrition|5": {
289 "acc": 0.7124183006535948,
290 "acc_stderr": 0.02591780611714716,
291 "acc_norm": 0.7124183006535948,
292 "acc_norm_stderr": 0.02591780611714716
293 },
294 "harness|hendrycksTest-philosophy|5": {
295 "acc": 0.7106109324758842,
296 "acc_stderr": 0.025755865922632945,
297 "acc_norm": 0.7106109324758842,
298 "acc_norm_stderr": 0.025755865922632945
299 },
300 "harness|hendrycksTest-prehistory|5": {
301 "acc": 0.6975308641975309,
302 "acc_stderr": 0.02555765398186806,
303 "acc_norm": 0.6975308641975309,
304 "acc_norm_stderr": 0.02555765398186806
305 },
306 "harness|hendrycksTest-professional_accounting|5": {
307 "acc": 0.49645390070921985,
308 "acc_stderr": 0.02982674915328092,
309 "acc_norm": 0.49645390070921985,
310 "acc_norm_stderr": 0.02982674915328092
311 },
312 "harness|hendrycksTest-professional_law|5": {
313 "acc": 0.4745762711864407,
314 "acc_stderr": 0.01275371692910101,
315 "acc_norm": 0.4745762711864407,
316 "acc_norm_stderr": 0.01275371692910101
317 },
318 "harness|hendrycksTest-professional_medicine|5": {
319 "acc": 0.6507352941176471,
320 "acc_stderr": 0.028959755196824862,
321 "acc_norm": 0.6507352941176471,
322 "acc_norm_stderr": 0.028959755196824862
323 },
324 "harness|hendrycksTest-professional_psychology|5": {
325 "acc": 0.6323529411764706,
326 "acc_stderr": 0.019506291693954843,
327 "acc_norm": 0.6323529411764706,
328 "acc_norm_stderr": 0.019506291693954843
329 },
330 "harness|hendrycksTest-public_relations|5": {
331 "acc": 0.6363636363636364,
332 "acc_stderr": 0.046075820907199756,
333 "acc_norm": 0.6363636363636364,
334 "acc_norm_stderr": 0.046075820907199756
335 },
336 "harness|hendrycksTest-security_studies|5": {
337 "acc": 0.7183673469387755,
338 "acc_stderr": 0.028795185574291293,
339 "acc_norm": 0.7183673469387755,
340 "acc_norm_stderr": 0.028795185574291293
341 },
342 "harness|hendrycksTest-sociology|5": {
343 "acc": 0.835820895522388,
344 "acc_stderr": 0.026193923544454125,
345 "acc_norm": 0.835820895522388,
346 "acc_norm_stderr": 0.026193923544454125
347 },
348 "harness|hendrycksTest-us_foreign_policy|5": {
349 "acc": 0.87,
350 "acc_stderr": 0.033799766898963086,
351 "acc_norm": 0.87,
352 "acc_norm_stderr": 0.033799766898963086
353 },
354 "harness|hendrycksTest-virology|5": {
355 "acc": 0.5180722891566265,
356 "acc_stderr": 0.03889951252827216,
357 "acc_norm": 0.5180722891566265,
358 "acc_norm_stderr": 0.03889951252827216
359 },
360 "harness|hendrycksTest-world_religions|5": {
361 "acc": 0.8187134502923976,
362 "acc_stderr": 0.029547741687640038,
363 "acc_norm": 0.8187134502923976,
364 "acc_norm_stderr": 0.029547741687640038
365 },
366 "harness|truthfulqa:mc|0": {
367 "mc1": 0.39657282741738065,
368 "mc1_stderr": 0.017124930942023518,
369 "mc2": 0.5688038233837539,
370 "mc2_stderr": 0.015263125204118244
371 },
372 "harness|winogrande|5": {
373 "acc": 0.7963693764798737,
374 "acc_stderr": 0.011317798781626918
375 },
376 "harness|gsm8k|5": {
377 "acc": 0.6103108415466262,
378 "acc_stderr": 0.01343312323611072
379 }
380}1!pip install -qU transformers accelerate
2
3from transformers import AutoTokenizer
4import transformers
5import torch
6
7model = "Isaak-Carter/J.O.S.I.E.3-Beta10-7B-slerp"
8messages = [{"role": "user", "content": "What is a large language model?"}]
9
10tokenizer = AutoTokenizer.from_pretrained(model)
11prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
12pipeline = transformers.pipeline(
13 "text-generation",
14 model=model,
15 torch_dtype=torch.float16,
16 device_map="auto",
17)
18
19outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
20print(outputs[0]["generated_text"])