base_model: google/gemma-2-2b
dtype: bfloat16
merge_method: ties
parameters:
  int8_mask: 1.0
  normalize: 1.0
slices:
- sources:
  - layer_range: [0, 26]
    model: google/gemma-2-2b
  - layer_range: [0, 26]
    model: google/gemma-2-2b-it
    parameters:
      density:
      - filter: model.layers.0.self_attn.q_proj
        value: 0.30462414915655517
      - filter: model.layers.1.self_attn.q_proj
        value: 0.5243267238828055
      - filter: model.layers.2.self_attn.q_proj
        value: 0.3365862681266647
      - filter: model.layers.3.self_attn.q_proj
        value: 0.39377774489493933
      - filter: model.layers.4.self_attn.q_proj
        value: 0.5428603137022787
      - filter: model.layers.5.self_attn.q_proj
        value: 0.49388132583604616
      - filter: model.layers.6.self_attn.q_proj
        value: 0.3083974548683042
      - filter: model.layers.7.self_attn.q_proj
        value: 0.39629328203610537
      - filter: model.layers.8.self_attn.q_proj
        value: 0.4097957975732465
      - filter: model.layers.9.self_attn.q_proj
        value: 0.5082346848179935
      - filter: model.layers.10.self_attn.q_proj
        value: 0.3383989345960343
      - filter: model.layers.11.self_attn.q_proj
        value: 0.4997262503699319
      - filter: model.layers.12.self_attn.q_proj
        value: 0.5750443918319029
      - filter: model.layers.13.self_attn.q_proj
        value: 0.43454424385912993
      - filter: model.layers.14.self_attn.q_proj
        value: 0.6742231429416987
      - filter: model.layers.15.self_attn.q_proj
        value: 0.34272713820656997
      - filter: model.layers.16.self_attn.q_proj
        value: 0.5344258656407221
      - filter: model.layers.17.self_attn.q_proj
        value: 0.4468999704054454
      - filter: model.layers.18.self_attn.q_proj
        value: 0.4482317253625333
      - filter: model.layers.19.self_attn.q_proj
        value: 0.5271382065699911
      - filter: model.layers.20.self_attn.q_proj
        value: 0.5559189109203906
      - filter: model.layers.21.self_attn.q_proj
        value: 0.5799644865344776
      - filter: model.layers.22.self_attn.q_proj
        value: 0.3292986090559337
      - filter: model.layers.23.self_attn.q_proj
        value: 0.33924977804084044
      - filter: model.layers.24.self_attn.q_proj
        value: 0.3134285291506363
      - filter: model.layers.25.self_attn.q_proj
        value: 0.31734980763539505
      - filter: model.layers.0.self_attn.k_proj
        value: 0.3053270198283516
      - filter: model.layers.1.self_attn.k_proj
        value: 0.43872447469665576
      - filter: model.layers.2.self_attn.k_proj
        value: 0.3261541876294762
      - filter: model.layers.3.self_attn.k_proj
        value: 0.3491639538324948
      - filter: model.layers.4.self_attn.k_proj
        value: 0.46465670316661734
      - filter: model.layers.5.self_attn.k_proj
        value: 0.4071322876590707
      - filter: model.layers.6.self_attn.k_proj
        value: 0.3575244155075466
      - filter: model.layers.7.self_attn.k_proj
        value: 0.44915655519384434
      - filter: model.layers.8.self_attn.k_proj
        value: 0.36721663213968625
      - filter: model.layers.9.self_attn.k_proj
        value: 0.4397232909144717
      - filter: model.layers.10.self_attn.k_proj
        value: 0.34609351879254213
      - filter: model.layers.11.self_attn.k_proj
        value: 0.5355356614382953
      - filter: model.layers.12.self_attn.k_proj
        value: 0.5992379402189997
      - filter: model.layers.13.self_attn.k_proj
        value: 0.47275821248890204
      - filter: model.layers.14.self_attn.k_proj
        value: 0.6111127552530333
      - filter: model.layers.15.self_attn.k_proj
        value: 0.4480837525895235
      - filter: model.layers.16.self_attn.k_proj
        value: 0.60378810298905
      - filter: model.layers.17.self_attn.k_proj
        value: 0.39666321396862975
      - filter: model.layers.18.self_attn.k_proj
        value: 0.44930452796685405
      - filter: model.layers.19.self_attn.k_proj
        value: 0.5763021604024859
      - filter: model.layers.20.self_attn.k_proj
        value: 0.5605800532701983
      - filter: model.layers.21.self_attn.k_proj
        value: 0.5279890500147972
      - filter: model.layers.22.self_attn.k_proj
        value: 0.3384729209825392
      - filter: model.layers.23.self_attn.k_proj
        value: 0.4068363421130512
      - filter: model.layers.24.self_attn.k_proj
        value: 0.34109943770346257
      - filter: model.layers.25.self_attn.k_proj
        value: 0.35659958567623556
      - filter: model.layers.0.self_attn.v_proj
        value: 0.3309263095590411
      - filter: model.layers.1.self_attn.v_proj
        value: 0.45344776561112754
      - filter: model.layers.2.self_attn.v_proj
        value: 0.30325540100621484
      - filter: model.layers.3.self_attn.v_proj
        value: 0.3882287659070731
      - filter: model.layers.4.self_attn.v_proj
        value: 0.5263983427049422
      - filter: model.layers.5.self_attn.v_proj
        value: 0.48196951760875995
      - filter: model.layers.6.self_attn.v_proj
        value: 0.3515315182006511
      - filter: model.layers.7.self_attn.v_proj
        value: 0.35756140870079905
      - filter: model.layers.8.self_attn.v_proj
        value: 0.4864456939923054
      - filter: model.layers.9.self_attn.v_proj
        value: 0.5143015685113939
      - filter: model.layers.10.self_attn.v_proj
        value: 0.3134655223438887
      - filter: model.layers.11.self_attn.v_proj
        value: 0.47061260728026044
      - filter: model.layers.12.self_attn.v_proj
        value: 0.5261023971589227
      - filter: model.layers.13.self_attn.v_proj
        value: 0.3451686889612311
      - filter: model.layers.14.self_attn.v_proj
        value: 0.6927197395679194
      - filter: model.layers.15.self_attn.v_proj
        value: 0.31124593074874224
      - filter: model.layers.16.self_attn.v_proj
        value: 0.52469665581533
      - filter: model.layers.17.self_attn.v_proj
        value: 0.5372003551346551
      - filter: model.layers.18.self_attn.v_proj
        value: 0.5149674459899378
      - filter: model.layers.19.self_attn.v_proj
        value: 0.425924829831311
      - filter: model.layers.20.self_attn.v_proj
        value: 0.5479653743711157
      - filter: model.layers.21.self_attn.v_proj
        value: 0.5006510802012429
      - filter: model.layers.22.self_attn.v_proj
        value: 0.3906333234684818
      - filter: model.layers.23.self_attn.v_proj
        value: 0.3831606984314886
      - filter: model.layers.24.self_attn.v_proj
        value: 0.33207309854986683
      - filter: model.layers.25.self_attn.v_proj
        value: 0.4645827167801124
      - filter: model.layers.0.self_attn.o_proj
        value: 0.319939331163066
      - filter: model.layers.1.self_attn.o_proj
        value: 0.4388724474696656
      - filter: model.layers.2.self_attn.o_proj
        value: 0.32907664989641905
      - filter: model.layers.3.self_attn.o_proj
        value: 0.381939923054158
      - filter: model.layers.4.self_attn.o_proj
        value: 0.3908552826279964
      - filter: model.layers.5.self_attn.o_proj
        value: 0.3873409292690145
      - filter: model.layers.6.self_attn.o_proj
        value: 0.3123927197395679
      - filter: model.layers.7.self_attn.o_proj
        value: 0.35800532701982835
      - filter: model.layers.8.self_attn.o_proj
        value: 0.36710565255992894
      - filter: model.layers.9.self_attn.o_proj
        value: 0.7862755253033442
      - filter: model.layers.10.self_attn.o_proj
        value: 0.45267090855282627
      - filter: model.layers.11.self_attn.o_proj
        value: 0.4594406629180231
      - filter: model.layers.12.self_attn.o_proj
        value: 0.6508804379994081
      - filter: model.layers.13.self_attn.o_proj
        value: 0.4068363421130512
      - filter: model.layers.14.self_attn.o_proj
        value: 0.6868008286475289
      - filter: model.layers.15.self_attn.o_proj
        value: 0.38671204498372297
      - filter: model.layers.16.self_attn.o_proj
        value: 0.7229431784551643
      - filter: model.layers.17.self_attn.o_proj
        value: 0.6892053862089376
      - filter: model.layers.18.self_attn.o_proj
        value: 0.39240899674459895
      - filter: model.layers.19.self_attn.o_proj
        value: 0.3546389464338562
      - filter: model.layers.20.self_attn.o_proj
        value: 0.3692512577685706
      - filter: model.layers.21.self_attn.o_proj
        value: 0.4096108316069843
      - filter: model.layers.22.self_attn.o_proj
        value: 0.3552678307191477
      - filter: model.layers.23.self_attn.o_proj
        value: 0.3398046759396271
      - filter: model.layers.24.self_attn.o_proj
        value: 0.3760949985202723
      - filter: model.layers.25.self_attn.o_proj
        value: 0.35097662030186444
      - filter: model.layers.0.mlp.gate_proj
        value: 0.7129920094702575
      - filter: model.layers.1.mlp.gate_proj
        value: 0.3369562000591891
      - filter: model.layers.2.mlp.gate_proj
        value: 0.3351805267830719
      - filter: model.layers.3.mlp.gate_proj
        value: 0.4020272269902338
      - filter: model.layers.4.mlp.gate_proj
        value: 0.6620893755548979
      - filter: model.layers.5.mlp.gate_proj
        value: 0.519258656407221
      - filter: model.layers.6.mlp.gate_proj
        value: 0.3784625628884285
      - filter: model.layers.7.mlp.gate_proj
        value: 0.467394199467298
      - filter: model.layers.8.mlp.gate_proj
        value: 0.4318067475584493
      - filter: model.layers.9.mlp.gate_proj
        value: 0.4784921574430305
      - filter: model.layers.10.mlp.gate_proj
        value: 0.4866306599585676
      - filter: model.layers.11.mlp.gate_proj
        value: 0.39166913287955013
      - filter: model.layers.12.mlp.gate_proj
        value: 0.363591299200947
      - filter: model.layers.13.mlp.gate_proj
        value: 0.3617786327315774
      - filter: model.layers.14.mlp.gate_proj
        value: 0.30514205386208937
      - filter: model.layers.15.mlp.gate_proj
        value: 0.3103580941106836
      - filter: model.layers.16.mlp.gate_proj
        value: 0.346833382657591
      - filter: model.layers.17.mlp.gate_proj
        value: 0.4897750813850251
      - filter: model.layers.18.mlp.gate_proj
        value: 0.3015167209233501
      - filter: model.layers.19.mlp.gate_proj
        value: 0.34561260728026044
      - filter: model.layers.20.mlp.gate_proj
        value: 0.4940662918023083
      - filter: model.layers.21.mlp.gate_proj
        value: 0.49628588339745483
      - filter: model.layers.22.mlp.gate_proj
        value: 0.3769458419650784
      - filter: model.layers.23.mlp.gate_proj
        value: 0.47560668836934006
      - filter: model.layers.24.mlp.gate_proj
        value: 0.6521012133767387
      - filter: model.layers.25.mlp.gate_proj
        value: 0.5526265167209233
      - filter: model.layers.0.mlp.up_proj
        value: 0.4759026339153596
      - filter: model.layers.1.mlp.up_proj
        value: 0.3598549866824504
      - filter: model.layers.2.mlp.up_proj
        value: 0.464212784847588
      - filter: model.layers.3.mlp.up_proj
        value: 0.49055193844332645
      - filter: model.layers.4.mlp.up_proj
        value: 0.6642349807635395
      - filter: model.layers.5.mlp.up_proj
        value: 0.5684965966262208
      - filter: model.layers.6.mlp.up_proj
        value: 0.38009026339153595
      - filter: model.layers.7.mlp.up_proj
        value: 0.3273379698135543
      - filter: model.layers.8.mlp.up_proj
        value: 0.5282480023675643
      - filter: model.layers.9.mlp.up_proj
        value: 0.4439035217519976
      - filter: model.layers.10.mlp.up_proj
        value: 0.45899674459899376
      - filter: model.layers.11.mlp.up_proj
        value: 0.4393163657886949
      - filter: model.layers.12.mlp.up_proj
        value: 0.39825392127848475
      - filter: model.layers.13.mlp.up_proj
        value: 0.31690588931636576
      - filter: model.layers.14.mlp.up_proj
        value: 0.38083012725658477
      - filter: model.layers.15.mlp.up_proj
        value: 0.3624445102101213
      - filter: model.layers.16.mlp.up_proj
        value: 0.4013983427049423
      - filter: model.layers.17.mlp.up_proj
        value: 0.30591891092039064
      - filter: model.layers.18.mlp.up_proj
        value: 0.37842556969517605
      - filter: model.layers.19.mlp.up_proj
        value: 0.37402337969813554
      - filter: model.layers.20.mlp.up_proj
        value: 0.4992453388576502
      - filter: model.layers.21.mlp.up_proj
        value: 0.4995042912104173
      - filter: model.layers.22.mlp.up_proj
        value: 0.40187925421722404
      - filter: model.layers.23.mlp.up_proj
        value: 0.41508582420834567
      - filter: model.layers.24.mlp.up_proj
        value: 0.6025303344184669
      - filter: model.layers.25.mlp.up_proj
        value: 0.5377552530334418
      - filter: model.layers.0.mlp.down_proj
        value: 0.4555563776265167
      - filter: model.layers.1.mlp.down_proj
        value: 0.36151968037881027
      - filter: model.layers.2.mlp.down_proj
        value: 0.5025377330571175
      - filter: model.layers.3.mlp.down_proj
        value: 0.42496300680674753
      - filter: model.layers.4.mlp.down_proj
        value: 0.40162030186445696
      - filter: model.layers.5.mlp.down_proj
        value: 0.40273009766203016
      - filter: model.layers.6.mlp.down_proj
        value: 0.3211231133471441
      - filter: model.layers.7.mlp.down_proj
        value: 0.609817993489198
      - filter: model.layers.8.mlp.down_proj
        value: 0.5351287363125186
      - filter: model.layers.9.mlp.down_proj
        value: 0.5279150636282923
      - filter: model.layers.10.mlp.down_proj
        value: 0.30314442142645753
      - filter: model.layers.11.mlp.down_proj
        value: 0.3
      - filter: model.layers.12.mlp.down_proj
        value: 0.3705090263391536
      - filter: model.layers.13.mlp.down_proj
        value: 0.4358759988162178
      - filter: model.layers.14.mlp.down_proj
        value: 0.3469443622373483
      - filter: model.layers.15.mlp.down_proj
        value: 0.30369931932524413
      - filter: model.layers.16.mlp.down_proj
        value: 0.3927789286771234
      - filter: model.layers.17.mlp.down_proj
        value: 0.4097957975732465
      - filter: model.layers.18.mlp.down_proj
        value: 0.5509988162178159
      - filter: model.layers.19.mlp.down_proj
        value: 0.34805415803492157
      - filter: model.layers.20.mlp.down_proj
        value: 0.35338117786327317
      - filter: model.layers.21.mlp.down_proj
        value: 0.372802604320805
      - filter: model.layers.22.mlp.down_proj
        value: 0.5202944658182894
      - filter: model.layers.23.mlp.down_proj
        value: 0.8
      - filter: model.layers.24.mlp.down_proj
        value: 0.7236460491269606
      - filter: model.layers.25.mlp.down_proj
        value: 0.40051050606688365
      - value: 1.0
      weight:
      - value: 1.0
tokenizer_source: union
vidhirambhia/test-gemma-mbpp

gemma_2b_0.3_0.8_generated_ties_0.5_sparsity-merged

Merge Details

Merge Method

Models Merged

Configuration

vidhirambhia/
test-gemma-mbpp
