Views
No views yet
mistralai/Mistral-7B-v0.1 model fine-tuned over 52k alpaca dataset1# pip install transformers==4.35.2
2import torch
3from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
4from transformers import pipeline
5
6model_id="MaziyarPanahi/Mistral-7B-Alpaca-52k-v0.1"
7tokenizer = AutoTokenizer.from_pretrained(model_id)
8
9streamer = TextStreamer(tokenizer)
10
11model = AutoModelForCausalLM.from_pretrained(
12 model_id,
13 torch_dtype=torch.float16,
14 device_map="auto",
15)
16
17pipe = pipeline(
18 "text-generation",
19 model=model,
20 tokenizer=tokenizer,
21 max_new_tokens=1024,
22 temperature=0.1,
23 do_sample=True,
24 top_p=0.95,
25 repetition_penalty=1.15,
26 return_full_text=False,
27 streamer=streamer
28)
29
30prompt = """Below is an instruction that describes a task. Write a response that appropriately completes the request.
31
32### Instruction:
33describe about pros and cons of docker system. Answer in bullet point
34
35### Response:
36"""
37
38res = pipe(prompt)[0]['generated_text']Below is an instruction that describes a task. Write a response that appropriately completes the request.
### Instruction:
describe about pros and cons of docker system. Answer in bullet point
### Response:
Pros of Docker System:
- Improved portability - Docker containers can be easily moved between different environments, making it easier to deploy applications across multiple platforms.
- Increased security - Containers are isolated from each other, which helps prevent malicious code from spreading throughout the system.
- Better resource utilization - Containers allow for better resource management by allowing users to run multiple applications on a single host without having to worry about conflicts or performance issues.
Cons of Docker System:
- Learning curve - It takes time to learn how to use Docker effectively, as there are many commands and concepts involved.
- Limited customization options - While Docker provides some basic configuration options, more advanced features such as network routing require additional tools.
- Performance overhead - Running multiple containers on a single host may result in slower performance due to increased memory usage.</s>1{
2 "all": {
3 "acc": 0.6309850839451187,
4 "acc_stderr": 0.032333688535613636,
5 "acc_norm": 0.6368691004374645,
6 "acc_norm_stderr": 0.03298401757997533,
7 "mc1": 0.29008567931456547,
8 "mc1_stderr": 0.01588623687420952,
9 "mc2": 0.41501661742948026,
10 "mc2_stderr": 0.014285902986671931
11 },
12 "harness|arc:challenge|25": {
13 "acc": 0.5750853242320819,
14 "acc_stderr": 0.014445698968520767,
15 "acc_norm": 0.6092150170648464,
16 "acc_norm_stderr": 0.01425856388051378
17 },
18 "harness|hellaswag|10": {
19 "acc": 0.6221868153754232,
20 "acc_stderr": 0.0048384969668239025,
21 "acc_norm": 0.8212507468631747,
22 "acc_norm_stderr": 0.0038235918141330347
23 },
24 "harness|hendrycksTest-abstract_algebra|5": {
25 "acc": 0.32,
26 "acc_stderr": 0.046882617226215034,
27 "acc_norm": 0.32,
28 "acc_norm_stderr": 0.046882617226215034
29 },
30 "harness|hendrycksTest-anatomy|5": {
31 "acc": 0.6,
32 "acc_stderr": 0.04232073695151589,
33 "acc_norm": 0.6,
34 "acc_norm_stderr": 0.04232073695151589
35 },
36 "harness|hendrycksTest-astronomy|5": {
37 "acc": 0.6447368421052632,
38 "acc_stderr": 0.038947344870133176,
39 "acc_norm": 0.6447368421052632,
40 "acc_norm_stderr": 0.038947344870133176
41 },
42 "harness|hendrycksTest-business_ethics|5": {
43 "acc": 0.57,
44 "acc_stderr": 0.04975698519562428,
45 "acc_norm": 0.57,
46 "acc_norm_stderr": 0.04975698519562428
47 },
48 "harness|hendrycksTest-clinical_knowledge|5": {
49 "acc": 0.6792452830188679,
50 "acc_stderr": 0.02872750295788027,
51 "acc_norm": 0.6792452830188679,
52 "acc_norm_stderr": 0.02872750295788027
53 },
54 "harness|hendrycksTest-college_biology|5": {
55 "acc": 0.7430555555555556,
56 "acc_stderr": 0.03653946969442099,
57 "acc_norm": 0.7430555555555556,
58 "acc_norm_stderr": 0.03653946969442099
59 },
60 "harness|hendrycksTest-college_chemistry|5": {
61 "acc": 0.49,
62 "acc_stderr": 0.05024183937956912,
63 "acc_norm": 0.49,
64 "acc_norm_stderr": 0.05024183937956912
65 },
66 "harness|hendrycksTest-college_computer_science|5": {
67 "acc": 0.56,
68 "acc_stderr": 0.04988876515698589,
69 "acc_norm": 0.56,
70 "acc_norm_stderr": 0.04988876515698589
71 },
72 "harness|hendrycksTest-college_mathematics|5": {
73 "acc": 0.36,
74 "acc_stderr": 0.048241815132442176,
75 "acc_norm": 0.36,
76 "acc_norm_stderr": 0.048241815132442176
77 },
78 "harness|hendrycksTest-college_medicine|5": {
79 "acc": 0.653179190751445,
80 "acc_stderr": 0.036291466701596636,
81 "acc_norm": 0.653179190751445,
82 "acc_norm_stderr": 0.036291466701596636
83 },
84 "harness|hendrycksTest-college_physics|5": {
85 "acc": 0.4019607843137255,
86 "acc_stderr": 0.048786087144669955,
87 "acc_norm": 0.4019607843137255,
88 "acc_norm_stderr": 0.048786087144669955
89 },
90 "harness|hendrycksTest-computer_security|5": {
91 "acc": 0.79,
92 "acc_stderr": 0.04093601807403326,
93 "acc_norm": 0.79,
94 "acc_norm_stderr": 0.04093601807403326
95 },
96 "harness|hendrycksTest-conceptual_physics|5": {
97 "acc": 0.5702127659574469,
98 "acc_stderr": 0.03236214467715564,
99 "acc_norm": 0.5702127659574469,
100 "acc_norm_stderr": 0.03236214467715564
101 },
102 "harness|hendrycksTest-econometrics|5": {
103 "acc": 0.49122807017543857,
104 "acc_stderr": 0.047028804320496165,
105 "acc_norm": 0.49122807017543857,
106 "acc_norm_stderr": 0.047028804320496165
107 },
108 "harness|hendrycksTest-electrical_engineering|5": {
109 "acc": 0.5862068965517241,
110 "acc_stderr": 0.04104269211806232,
111 "acc_norm": 0.5862068965517241,
112 "acc_norm_stderr": 0.04104269211806232
113 },
114 "harness|hendrycksTest-elementary_mathematics|5": {
115 "acc": 0.3915343915343915,
116 "acc_stderr": 0.025138091388851116,
117 "acc_norm": 0.3915343915343915,
118 "acc_norm_stderr": 0.025138091388851116
119 },
120 "harness|hendrycksTest-formal_logic|5": {
121 "acc": 0.4444444444444444,
122 "acc_stderr": 0.04444444444444449,
123 "acc_norm": 0.4444444444444444,
124 "acc_norm_stderr": 0.04444444444444449
125 },
126 "harness|hendrycksTest-global_facts|5": {
127 "acc": 0.32,
128 "acc_stderr": 0.04688261722621504,
129 "acc_norm": 0.32,
130 "acc_norm_stderr": 0.04688261722621504
131 },
132 "harness|hendrycksTest-high_school_biology|5": {
133 "acc": 0.7419354838709677,
134 "acc_stderr": 0.02489246917246283,
135 "acc_norm": 0.7419354838709677,
136 "acc_norm_stderr": 0.02489246917246283
137 },
138 "harness|hendrycksTest-high_school_chemistry|5": {
139 "acc": 0.5024630541871922,
140 "acc_stderr": 0.035179450386910616,
141 "acc_norm": 0.5024630541871922,
142 "acc_norm_stderr": 0.035179450386910616
143 },
144 "harness|hendrycksTest-high_school_computer_science|5": {
145 "acc": 0.67,
146 "acc_stderr": 0.047258156262526066,
147 "acc_norm": 0.67,
148 "acc_norm_stderr": 0.047258156262526066
149 },
150 "harness|hendrycksTest-high_school_european_history|5": {
151 "acc": 0.7575757575757576,
152 "acc_stderr": 0.03346409881055953,
153 "acc_norm": 0.7575757575757576,
154 "acc_norm_stderr": 0.03346409881055953
155 },
156 "harness|hendrycksTest-high_school_geography|5": {
157 "acc": 0.7929292929292929,
158 "acc_stderr": 0.028869778460267042,
159 "acc_norm": 0.7929292929292929,
160 "acc_norm_stderr": 0.028869778460267042
161 },
162 "harness|hendrycksTest-high_school_government_and_politics|5": {
163 "acc": 0.8601036269430051,
164 "acc_stderr": 0.025033870583015184,
165 "acc_norm": 0.8601036269430051,
166 "acc_norm_stderr": 0.025033870583015184
167 },
168 "harness|hendrycksTest-high_school_macroeconomics|5": {
169 "acc": 0.6358974358974359,
170 "acc_stderr": 0.024396672985094764,
171 "acc_norm": 0.6358974358974359,
172 "acc_norm_stderr": 0.024396672985094764
173 },
174 "harness|hendrycksTest-high_school_mathematics|5": {
175 "acc": 0.362962962962963,
176 "acc_stderr": 0.029318203645206865,
177 "acc_norm": 0.362962962962963,
178 "acc_norm_stderr": 0.029318203645206865
179 },
180 "harness|hendrycksTest-high_school_microeconomics|5": {
181 "acc": 0.6218487394957983,
182 "acc_stderr": 0.03149930577784906,
183 "acc_norm": 0.6218487394957983,
184 "acc_norm_stderr": 0.03149930577784906
185 },
186 "harness|hendrycksTest-high_school_physics|5": {
187 "acc": 0.32450331125827814,
188 "acc_stderr": 0.038227469376587525,
189 "acc_norm": 0.32450331125827814,
190 "acc_norm_stderr": 0.038227469376587525
191 },
192 "harness|hendrycksTest-high_school_psychology|5": {
193 "acc": 0.8146788990825689,
194 "acc_stderr": 0.016659279700295838,
195 "acc_norm": 0.8146788990825689,
196 "acc_norm_stderr": 0.016659279700295838
197 },
198 "harness|hendrycksTest-high_school_statistics|5": {
199 "acc": 0.49537037037037035,
200 "acc_stderr": 0.03409825519163572,
201 "acc_norm": 0.49537037037037035,
202 "acc_norm_stderr": 0.03409825519163572
203 },
204 "harness|hendrycksTest-high_school_us_history|5": {
205 "acc": 0.7892156862745098,
206 "acc_stderr": 0.028626547912437406,
207 "acc_norm": 0.7892156862745098,
208 "acc_norm_stderr": 0.028626547912437406
209 },
210 "harness|hendrycksTest-high_school_world_history|5": {
211 "acc": 0.7552742616033755,
212 "acc_stderr": 0.027985699387036423,
213 "acc_norm": 0.7552742616033755,
214 "acc_norm_stderr": 0.027985699387036423
215 },
216 "harness|hendrycksTest-human_aging|5": {
217 "acc": 0.6636771300448431,
218 "acc_stderr": 0.031708824268455,
219 "acc_norm": 0.6636771300448431,
220 "acc_norm_stderr": 0.031708824268455
221 },
222 "harness|hendrycksTest-human_sexuality|5": {
223 "acc": 0.7862595419847328,
224 "acc_stderr": 0.0359546161177469,
225 "acc_norm": 0.7862595419847328,
226 "acc_norm_stderr": 0.0359546161177469
227 },
228 "harness|hendrycksTest-international_law|5": {
229 "acc": 0.7933884297520661,
230 "acc_stderr": 0.03695980128098824,
231 "acc_norm": 0.7933884297520661,
232 "acc_norm_stderr": 0.03695980128098824
233 },
234 "harness|hendrycksTest-jurisprudence|5": {
235 "acc": 0.7592592592592593,
236 "acc_stderr": 0.04133119440243838,
237 "acc_norm": 0.7592592592592593,
238 "acc_norm_stderr": 0.04133119440243838
239 },
240 "harness|hendrycksTest-logical_fallacies|5": {
241 "acc": 0.803680981595092,
242 "acc_stderr": 0.031207970394709218,
243 "acc_norm": 0.803680981595092,
244 "acc_norm_stderr": 0.031207970394709218
245 },
246 "harness|hendrycksTest-machine_learning|5": {
247 "acc": 0.5178571428571429,
248 "acc_stderr": 0.047427623612430116,
249 "acc_norm": 0.5178571428571429,
250 "acc_norm_stderr": 0.047427623612430116
251 },
252 "harness|hendrycksTest-management|5": {
253 "acc": 0.8252427184466019,
254 "acc_stderr": 0.03760178006026621,
255 "acc_norm": 0.8252427184466019,
256 "acc_norm_stderr": 0.03760178006026621
257 },
258 "harness|hendrycksTest-marketing|5": {
259 "acc": 0.8632478632478633,
260 "acc_stderr": 0.022509033937077816,
261 "acc_norm": 0.8632478632478633,
262 "acc_norm_stderr": 0.022509033937077816
263 },
264 "harness|hendrycksTest-medical_genetics|5": {
265 "acc": 0.74,
266 "acc_stderr": 0.04408440022768078,
267 "acc_norm": 0.74,
268 "acc_norm_stderr": 0.04408440022768078
269 },
270 "harness|hendrycksTest-miscellaneous|5": {
271 "acc": 0.8173690932311622,
272 "acc_stderr": 0.013816335389973136,
273 "acc_norm": 0.8173690932311622,
274 "acc_norm_stderr": 0.013816335389973136
275 },
276 "harness|hendrycksTest-moral_disputes|5": {
277 "acc": 0.7023121387283237,
278 "acc_stderr": 0.024617055388677,
279 "acc_norm": 0.7023121387283237,
280 "acc_norm_stderr": 0.024617055388677
281 },
282 "harness|hendrycksTest-moral_scenarios|5": {
283 "acc": 0.2335195530726257,
284 "acc_stderr": 0.014149575348976269,
285 "acc_norm": 0.2335195530726257,
286 "acc_norm_stderr": 0.014149575348976269
287 },
288 "harness|hendrycksTest-nutrition|5": {
289 "acc": 0.7450980392156863,
290 "acc_stderr": 0.024954184324879905,
291 "acc_norm": 0.7450980392156863,
292 "acc_norm_stderr": 0.024954184324879905
293 },
294 "harness|hendrycksTest-philosophy|5": {
295 "acc": 0.7106109324758842,
296 "acc_stderr": 0.025755865922632945,
297 "acc_norm": 0.7106109324758842,
298 "acc_norm_stderr": 0.025755865922632945
299 },
300 "harness|hendrycksTest-prehistory|5": {
301 "acc": 0.7191358024691358,
302 "acc_stderr": 0.025006469755799215,
303 "acc_norm": 0.7191358024691358,
304 "acc_norm_stderr": 0.025006469755799215
305 },
306 "harness|hendrycksTest-professional_accounting|5": {
307 "acc": 0.4716312056737589,
308 "acc_stderr": 0.029779450957303062,
309 "acc_norm": 0.4716312056737589,
310 "acc_norm_stderr": 0.029779450957303062
311 },
312 "harness|hendrycksTest-professional_law|5": {
313 "acc": 0.4498044328552803,
314 "acc_stderr": 0.012705721498565107,
315 "acc_norm": 0.4498044328552803,
316 "acc_norm_stderr": 0.012705721498565107
317 },
318 "harness|hendrycksTest-professional_medicine|5": {
319 "acc": 0.6580882352941176,
320 "acc_stderr": 0.02881472242225418,
321 "acc_norm": 0.6580882352941176,
322 "acc_norm_stderr": 0.02881472242225418
323 },
324 "harness|hendrycksTest-professional_psychology|5": {
325 "acc": 0.6519607843137255,
326 "acc_stderr": 0.019270998708223974,
327 "acc_norm": 0.6519607843137255,
328 "acc_norm_stderr": 0.019270998708223974
329 },
330 "harness|hendrycksTest-public_relations|5": {
331 "acc": 0.6636363636363637,
332 "acc_stderr": 0.04525393596302506,
333 "acc_norm": 0.6636363636363637,
334 "acc_norm_stderr": 0.04525393596302506
335 },
336 "harness|hendrycksTest-security_studies|5": {
337 "acc": 0.7224489795918367,
338 "acc_stderr": 0.028666857790274645,
339 "acc_norm": 0.7224489795918367,
340 "acc_norm_stderr": 0.028666857790274645
341 },
342 "harness|hendrycksTest-sociology|5": {
343 "acc": 0.8557213930348259,
344 "acc_stderr": 0.02484575321230604,
345 "acc_norm": 0.8557213930348259,
346 "acc_norm_stderr": 0.02484575321230604
347 },
348 "harness|hendrycksTest-us_foreign_policy|5": {
349 "acc": 0.86,
350 "acc_stderr": 0.03487350880197771,
351 "acc_norm": 0.86,
352 "acc_norm_stderr": 0.03487350880197771
353 },
354 "harness|hendrycksTest-virology|5": {
355 "acc": 0.5481927710843374,
356 "acc_stderr": 0.03874371556587953,
357 "acc_norm": 0.5481927710843374,
358 "acc_norm_stderr": 0.03874371556587953
359 },
360 "harness|hendrycksTest-world_religions|5": {
361 "acc": 0.8421052631578947,
362 "acc_stderr": 0.027966785859160896,
363 "acc_norm": 0.8421052631578947,
364 "acc_norm_stderr": 0.027966785859160896
365 },
366 "harness|truthfulqa:mc|0": {
367 "mc1": 0.29008567931456547,
368 "mc1_stderr": 0.01588623687420952,
369 "mc2": 0.41501661742948026,
370 "mc2_stderr": 0.014285902986671931
371 },
372 "harness|winogrande|5": {
373 "acc": 0.7734806629834254,
374 "acc_stderr": 0.011764149054698332
375 },
376 "harness|gsm8k|5": {
377 "acc": 0.37452615617892343,
378 "acc_stderr": 0.013331774158491393
379 }
380}| Metric | Value |
|---|---|
| Avg. | 60.46 |
| AI2 Reasoning Challenge (25-Shot) | 60.92 |
| HellaSwag (10-Shot) | 82.13 |
| MMLU (5-Shot) | 63.41 |
| TruthfulQA (0-shot) | 41.50 |
| Winogrande (5-shot) | 77.35 |
| GSM8k (5-shot) | 37.45 |