Views
No views yet
1base_model: ./Yosegi-0603
2dtype: bfloat16
3merge_method: breadcrumbs_ties
4parameters:
5 int8_mask: 1.0
6 normalize: 0.0
7slices:
8- sources:
9 - layer_range: [0, 2]
10 model: ./Yosegi-0601
11 parameters:
12 density:
13 - filter: self_attn
14 value: 1.0
15 - filter: mlp
16 value: 1.0
17 - value: 1.0
18 gamma:
19 - filter: self_attn
20 value: -0.050387850856855765
21 - filter: mlp
22 value: -0.17075015661203768
23 - value: -0.008041653902986862
24 weight:
25 - filter: self_attn
26 value: 0.0999312941470471
27 - filter: mlp
28 value: 0.541727762184749
29 - value: 0.6837012779994258
30 - layer_range: [0, 2]
31 model: ./Ninja-2B_JP
32 parameters:
33 density:
34 - filter: self_attn
35 value: 0.8218846237599902
36 - filter: mlp
37 value: 1.0
38 - value: 0.9254078866667358
39 gamma:
40 - filter: self_attn
41 value: -0.11213758231875963
42 - filter: mlp
43 value: 0.021586098873668948
44 - value: -0.12827998218659437
45 weight:
46 - filter: self_attn
47 value: 0.40391646444657003
48 - filter: mlp
49 value: 0.623121864641881
50 - value: 0.5967833694632534
51 - layer_range: [0, 2]
52 model: ./Yosegi-0603
53- sources:
54 - layer_range: [2, 4]
55 model: ./Yosegi-0601
56 parameters:
57 density:
58 - filter: self_attn
59 value: 0.8079479346300947
60 - filter: mlp
61 value: 1.0
62 - value: 0.710146185559419
63 gamma:
64 - filter: self_attn
65 value: 0.1383609589681566
66 - filter: mlp
67 value: 0.21188532059635062
68 - value: 0.2994723556443468
69 weight:
70 - filter: self_attn
71 value: 0.48107070906079974
72 - filter: mlp
73 value: 0.5848073552919492
74 - value: 0.4583842493359253
75 - layer_range: [2, 4]
76 model: ./Ninja-2B_JP
77 parameters:
78 density:
79 - filter: self_attn
80 value: 1.0
81 - filter: mlp
82 value: 0.934378153535579
83 - value: 1.0
84 gamma:
85 - filter: self_attn
86 value: 0.073192612278188
87 - filter: mlp
88 value: 0.07939126555063317
89 - value: -0.06891845030175699
90 weight:
91 - filter: self_attn
92 value: 0.32120386994101
93 - filter: mlp
94 value: 0.5001108459121922
95 - value: 0.9138710221666694
96 - layer_range: [2, 4]
97 model: ./Yosegi-0603
98- sources:
99 - layer_range: [4, 6]
100 model: ./Yosegi-0601
101 parameters:
102 density:
103 - filter: self_attn
104 value: 1.0
105 - filter: mlp
106 value: 0.7237519222177541
107 - value: 0.776951124863642
108 gamma:
109 - filter: self_attn
110 value: -0.2265121048274062
111 - filter: mlp
112 value: -0.1757947421960496
113 - value: -0.11401593728931929
114 weight:
115 - filter: self_attn
116 value: 0.6448742737026658
117 - filter: mlp
118 value: 0.13809748641457986
119 - value: 0.3950550285769662
120 - layer_range: [4, 6]
121 model: ./Ninja-2B_JP
122 parameters:
123 density:
124 - filter: self_attn
125 value: 0.9649359194114893
126 - filter: mlp
127 value: 0.916637032428399
128 - value: 1.0
129 gamma:
130 - filter: self_attn
131 value: -0.16291684846287688
132 - filter: mlp
133 value: -0.19013548712121703
134 - value: 0.038409066391918795
135 weight:
136 - filter: self_attn
137 value: 0.1977358472772336
138 - filter: mlp
139 value: 0.22661167907612348
140 - value: 0.6426575016448257
141 - layer_range: [4, 6]
142 model: ./Yosegi-0603
143- sources:
144 - layer_range: [6, 8]
145 model: ./Yosegi-0601
146 parameters:
147 density:
148 - filter: self_attn
149 value: 0.8727809666891416
150 - filter: mlp
151 value: 1.0
152 - value: 0.5160677785559116
153 gamma:
154 - filter: self_attn
155 value: 0.14245180617134273
156 - filter: mlp
157 value: 0.08189992601998919
158 - value: -0.1038827997670827
159 weight:
160 - filter: self_attn
161 value: 0.23575676914257698
162 - filter: mlp
163 value: 0.4047231670507743
164 - value: 0.34207794631274374
165 - layer_range: [6, 8]
166 model: ./Ninja-2B_JP
167 parameters:
168 density:
169 - filter: self_attn
170 value: 1.0
171 - filter: mlp
172 value: 1.0
173 - value: 1.0
174 gamma:
175 - filter: self_attn
176 value: 0.576775501046583
177 - filter: mlp
178 value: -0.046028636298718645
179 - value: -0.024161321403060265
180 weight:
181 - filter: self_attn
182 value: 0.833089842843994
183 - filter: mlp
184 value: 0.5434667434613458
185 - value: 0.2946693008513797
186 - layer_range: [6, 8]
187 model: ./Yosegi-0603
188- sources:
189 - layer_range: [8, 10]
190 model: ./Yosegi-0601
191 parameters:
192 density:
193 - filter: self_attn
194 value: 1.0
195 - filter: mlp
196 value: 1.0
197 - value: 0.9930269337531187
198 gamma:
199 - filter: self_attn
200 value: 0.4549980941970383
201 - filter: mlp
202 value: 0.10362988739411173
203 - value: -0.43800391668559174
204 weight:
205 - filter: self_attn
206 value: 0.19663450954683193
207 - filter: mlp
208 value: 0.16783989984505265
209 - value: 0.7465091417598162
210 - layer_range: [8, 10]
211 model: ./Ninja-2B_JP
212 parameters:
213 density:
214 - filter: self_attn
215 value: 0.797370597380894
216 - filter: mlp
217 value: 1.0
218 - value: 1.0
219 gamma:
220 - filter: self_attn
221 value: -0.0665958634205702
222 - filter: mlp
223 value: -0.058297473060129834
224 - value: -0.38206760673090134
225 weight:
226 - filter: self_attn
227 value: 0.7015967347604024
228 - filter: mlp
229 value: 0.7733694864324641
230 - value: 0.7636921732342238
231 - layer_range: [8, 10]
232 model: ./Yosegi-0603
233- sources:
234 - layer_range: [10, 12]
235 model: ./Yosegi-0601
236 parameters:
237 density:
238 - filter: self_attn
239 value: 0.8047576867589878
240 - filter: mlp
241 value: 0.8852533319203653
242 - value: 0.7707342647603538
243 gamma:
244 - filter: self_attn
245 value: -0.054343999574509694
246 - filter: mlp
247 value: -0.3465154355167133
248 - value: 0.022315854655582765
249 weight:
250 - filter: self_attn
251 value: 0.4396484757291151
252 - filter: mlp
253 value: 0.34318396468602314
254 - value: 0.8236034746664869
255 - layer_range: [10, 12]
256 model: ./Ninja-2B_JP
257 parameters:
258 density:
259 - filter: self_attn
260 value: 0.9058471193805165
261 - filter: mlp
262 value: 1.0
263 - value: 1.0
264 gamma:
265 - filter: self_attn
266 value: 0.1221058588826469
267 - filter: mlp
268 value: -0.4004985640890659
269 - value: 0.3219195440395816
270 weight:
271 - filter: self_attn
272 value: 0.3565443612269864
273 - filter: mlp
274 value: 0.2817057075232181
275 - value: 0.5934890337808251
276 - layer_range: [10, 12]
277 model: ./Yosegi-0603
278- sources:
279 - layer_range: [12, 14]
280 model: ./Yosegi-0601
281 parameters:
282 density:
283 - filter: self_attn
284 value: 1.0
285 - filter: mlp
286 value: 1.0
287 - value: 1.0
288 gamma:
289 - filter: self_attn
290 value: -0.027897116191693133
291 - filter: mlp
292 value: -0.1765379388255607
293 - value: 0.09108936063176161
294 weight:
295 - filter: self_attn
296 value: 0.4499753137521779
297 - filter: mlp
298 value: 0.901296236087911
299 - value: 0.3548680126954006
300 - layer_range: [12, 14]
301 model: ./Ninja-2B_JP
302 parameters:
303 density:
304 - filter: self_attn
305 value: 0.8973815150776497
306 - filter: mlp
307 value: 0.6029953465961999
308 - value: 1.0
309 gamma:
310 - filter: self_attn
311 value: 0.10393082898402586
312 - filter: mlp
313 value: 0.15993577688878796
314 - value: 0.011410411917833683
315 weight:
316 - filter: self_attn
317 value: 0.2211644023056492
318 - filter: mlp
319 value: 0.5677387594231849
320 - value: 0.1316535663010981
321 - layer_range: [12, 14]
322 model: ./Yosegi-0603
323- sources:
324 - layer_range: [14, 16]
325 model: ./Yosegi-0601
326 parameters:
327 density:
328 - filter: self_attn
329 value: 0.9584597245055072
330 - filter: mlp
331 value: 1.0
332 - value: 1.0
333 gamma:
334 - filter: self_attn
335 value: -0.17789727632680347
336 - filter: mlp
337 value: 0.2182263440314275
338 - value: 0.1449547656126498
339 weight:
340 - filter: self_attn
341 value: 0.4551004762874224
342 - filter: mlp
343 value: 0.9182082826762857
344 - value: 0.3736989395186422
345 - layer_range: [14, 16]
346 model: ./Ninja-2B_JP
347 parameters:
348 density:
349 - filter: self_attn
350 value: 0.7414465107848625
351 - filter: mlp
352 value: 1.0
353 - value: 0.7894887419395906
354 gamma:
355 - filter: self_attn
356 value: -0.07343933395880992
357 - filter: mlp
358 value: 0.250800731630588
359 - value: -0.2948778134297542
360 weight:
361 - filter: self_attn
362 value: 0.43125199001016495
363 - filter: mlp
364 value: 0.6182726353394477
365 - value: 0.838902157446268
366 - layer_range: [14, 16]
367 model: ./Yosegi-0603
368- sources:
369 - layer_range: [16, 18]
370 model: ./Yosegi-0601
371 parameters:
372 density:
373 - filter: self_attn
374 value: 0.9474287877268394
375 - filter: mlp
376 value: 1.0
377 - value: 0.9613380133344519
378 gamma:
379 - filter: self_attn
380 value: -0.08608895546593046
381 - filter: mlp
382 value: -0.07275416053291164
383 - value: -0.5796137860399382
384 weight:
385 - filter: self_attn
386 value: 0.5593420897751296
387 - filter: mlp
388 value: 0.7339447992880666
389 - value: 0.5447558586689005
390 - layer_range: [16, 18]
391 model: ./Ninja-2B_JP
392 parameters:
393 density:
394 - filter: self_attn
395 value: 0.9321536960575384
396 - filter: mlp
397 value: 1.0
398 - value: 0.9613033408813294
399 gamma:
400 - filter: self_attn
401 value: 0.20610728738224296
402 - filter: mlp
403 value: 0.2002206706624053
404 - value: -0.45349278793293785
405 weight:
406 - filter: self_attn
407 value: 0.16162975594196963
408 - filter: mlp
409 value: 0.21262726992327483
410 - value: 0.061213622827234075
411 - layer_range: [16, 18]
412 model: ./Yosegi-0603
413- sources:
414 - layer_range: [18, 20]
415 model: ./Yosegi-0601
416 parameters:
417 density:
418 - filter: self_attn
419 value: 1.0
420 - filter: mlp
421 value: 1.0
422 - value: 1.0
423 gamma:
424 - filter: self_attn
425 value: 0.03922456593148313
426 - filter: mlp
427 value: 0.3318035822806869
428 - value: -0.10373990685028205
429 weight:
430 - filter: self_attn
431 value: 0.8254441016674987
432 - filter: mlp
433 value: 0.4568039342431161
434 - value: 0.3152648515747969
435 - layer_range: [18, 20]
436 model: ./Ninja-2B_JP
437 parameters:
438 density:
439 - filter: self_attn
440 value: 1.0
441 - filter: mlp
442 value: 1.0
443 - value: 0.9807358937293073
444 gamma:
445 - filter: self_attn
446 value: -0.22734036563128657
447 - filter: mlp
448 value: 0.26113222150270854
449 - value: 0.17739039022957015
450 weight:
451 - filter: self_attn
452 value: 0.33759130475641996
453 - filter: mlp
454 value: 0.616639215544168
455 - value: 0.47560658618977714
456 - layer_range: [18, 20]
457 model: ./Yosegi-0603
458- sources:
459 - layer_range: [20, 22]
460 model: ./Yosegi-0601
461 parameters:
462 density:
463 - filter: self_attn
464 value: 0.9394514442960196
465 - filter: mlp
466 value: 1.0
467 - value: 0.9885037757465567
468 gamma:
469 - filter: self_attn
470 value: -0.17365709450334324
471 - filter: mlp
472 value: 0.0712279381144505
473 - value: 0.11809665485306464
474 weight:
475 - filter: self_attn
476 value: 0.485610337254665
477 - filter: mlp
478 value: 0.8406593173801935
479 - value: 0.5024102481819739
480 - layer_range: [20, 22]
481 model: ./Ninja-2B_JP
482 parameters:
483 density:
484 - filter: self_attn
485 value: 1.0
486 - filter: mlp
487 value: 1.0
488 - value: 1.0
489 gamma:
490 - filter: self_attn
491 value: -0.09980202641768818
492 - filter: mlp
493 value: 0.051454493742856926
494 - value: 0.14619126408666103
495 weight:
496 - filter: self_attn
497 value: 0.54772456079406
498 - filter: mlp
499 value: 0.3440893571099615
500 - value: 0.3747271233512448
501 - layer_range: [20, 22]
502 model: ./Yosegi-0603
503- sources:
504 - layer_range: [22, 24]
505 model: ./Yosegi-0601
506 parameters:
507 density:
508 - filter: self_attn
509 value: 1.0
510 - filter: mlp
511 value: 0.9474712362889293
512 - value: 1.0
513 gamma:
514 - filter: self_attn
515 value: -0.16020032978118146
516 - filter: mlp
517 value: -0.025085248873309034
518 - value: 0.06046174910893976
519 weight:
520 - filter: self_attn
521 value: 0.8654189362345427
522 - filter: mlp
523 value: 0.6344956382288498
524 - value: 0.6383979001549549
525 - layer_range: [22, 24]
526 model: ./Ninja-2B_JP
527 parameters:
528 density:
529 - filter: self_attn
530 value: 0.8240762427167851
531 - filter: mlp
532 value: 1.0
533 - value: 0.9004913821398048
534 gamma:
535 - filter: self_attn
536 value: -0.12224186789525764
537 - filter: mlp
538 value: -0.25877585460700525
539 - value: 0.35149388360871714
540 weight:
541 - filter: self_attn
542 value: 0.4294356408713786
543 - filter: mlp
544 value: 0.3920647298630233
545 - value: 0.795891295390721
546 - layer_range: [22, 24]
547 model: ./Yosegi-0603
548- sources:
549 - layer_range: [24, 26]
550 model: ./Yosegi-0601
551 parameters:
552 density:
553 - filter: self_attn
554 value: 1.0
555 - filter: mlp
556 value: 1.0
557 - value: 1.0
558 gamma:
559 - filter: self_attn
560 value: 0.16915580088030202
561 - filter: mlp
562 value: 0.2602652727555053
563 - value: 0.16985672723305376
564 weight:
565 - filter: self_attn
566 value: 0.420377024485687
567 - filter: mlp
568 value: 0.3401141209432324
569 - value: 0.4953511256159331
570 - layer_range: [24, 26]
571 model: ./Ninja-2B_JP
572 parameters:
573 density:
574 - filter: self_attn
575 value: 0.7290652609253236
576 - filter: mlp
577 value: 1.0
578 - value: 1.0
579 gamma:
580 - filter: self_attn
581 value: -0.1039167464696765
582 - filter: mlp
583 value: -0.18476572570059685
584 - value: 0.1221387313921081
585 weight:
586 - filter: self_attn
587 value: 0.2925002157134928
588 - filter: mlp
589 value: 0.3854740639588027
590 - value: 0.555448110317977
591 - layer_range: [24, 26]
592 model: ./Yosegi-0603
593- sources:
594 - layer_range: [26, 28]
595 model: ./Yosegi-0601
596 parameters:
597 density:
598 - filter: self_attn
599 value: 1.0
600 - filter: mlp
601 value: 0.9104496350690235
602 - value: 1.0
603 gamma:
604 - filter: self_attn
605 value: 0.24831264214235005
606 - filter: mlp
607 value: -0.03903149241855605
608 - value: 0.14189425093398259
609 weight:
610 - filter: self_attn
611 value: 0.7685811138035815
612 - filter: mlp
613 value: 0.06535011571274918
614 - value: 0.696502559577317
615 - layer_range: [26, 28]
616 model: ./Ninja-2B_JP
617 parameters:
618 density:
619 - filter: self_attn
620 value: 0.9236218028490522
621 - filter: mlp
622 value: 1.0
623 - value: 1.0
624 gamma:
625 - filter: self_attn
626 value: -0.2451400735890047
627 - filter: mlp
628 value: -0.21555851418482214
629 - value: 0.020418471695148876
630 weight:
631 - filter: self_attn
632 value: 0.451368534421561
633 - filter: mlp
634 value: 0.27412879847687055
635 - value: 0.18339776770537336
636 - layer_range: [26, 28]
637 model: ./Yosegi-0603
638- sources:
639 - layer_range: [28, 30]
640 model: ./Yosegi-0601
641 parameters:
642 density:
643 - filter: self_attn
644 value: 0.8590812961904566
645 - filter: mlp
646 value: 1.0
647 - value: 1.0
648 gamma:
649 - filter: self_attn
650 value: -0.06934549536310654
651 - filter: mlp
652 value: -0.28464693250998063
653 - value: -0.0588491947891552
654 weight:
655 - filter: self_attn
656 value: 0.26716389671655294
657 - filter: mlp
658 value: 0.8228280162386532
659 - value: 0.24197568479527135
660 - layer_range: [28, 30]
661 model: ./Ninja-2B_JP
662 parameters:
663 density:
664 - filter: self_attn
665 value: 0.7277181780542642
666 - filter: mlp
667 value: 0.74166025738732
668 - value: 1.0
669 gamma:
670 - filter: self_attn
671 value: 0.1772650150670655
672 - filter: mlp
673 value: 0.06545031487123437
674 - value: -0.28681451125993446
675 weight:
676 - filter: self_attn
677 value: 0.5781944040541174
678 - filter: mlp
679 value: 0.2288692970435767
680 - value: 0.689751088930503
681 - layer_range: [28, 30]
682 model: ./Yosegi-0603
683- sources:
684 - layer_range: [30, 32]
685 model: ./Yosegi-0601
686 parameters:
687 density:
688 - filter: self_attn
689 value: 0.8177341862620365
690 - filter: mlp
691 value: 0.8875629677599377
692 - value: 1.0
693 gamma:
694 - filter: self_attn
695 value: -0.06572527259889459
696 - filter: mlp
697 value: -0.18979543285938766
698 - value: -0.24122036571646263
699 weight:
700 - filter: self_attn
701 value: 0.5818433594657613
702 - filter: mlp
703 value: 0.36676821100234736
704 - value: 0.3580688869263428
705 - layer_range: [30, 32]
706 model: ./Ninja-2B_JP
707 parameters:
708 density:
709 - filter: self_attn
710 value: 0.8306036003344672
711 - filter: mlp
712 value: 0.6993970248745297
713 - value: 1.0
714 gamma:
715 - filter: self_attn
716 value: -0.20599853236581384
717 - filter: mlp
718 value: -0.2001187634455465
719 - value: -0.07654635090020837
720 weight:
721 - filter: self_attn
722 value: 0.37120677279712305
723 - filter: mlp
724 value: 0.13105486609905853
725 - value: 0.7204857820148367
726 - layer_range: [30, 32]
727 model: ./Yosegi-0603
728tokenizer_source: union