fridayfringe/

Tinydeepseek

  (ln1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)

  (attn): MLA(

    (compress_kv): Linear(in_features=256, out_features=64, bias=False)
    (kv_proj): Linear(in_features=64, out_features=512, bias=False)
    (compress_q): Linear(in_features=256, out_features=64, bias=False)
    (q_proj): Linear(in_features=64, out_features=256, bias=False)
    (q_rope): Linear(in_features=64, out_features=256, bias=False)
    (k_rope): Linear(in_features=64, out_features=32, bias=False)
    (out_proj): Linear(in_features=256, out_features=256, bias=False)
    (attn_dropout): Dropout(p=0.1, inplace=False)
    
  )

  (ln2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)

  (ffn): MOELayer(
    (experts): ModuleList(
      (0-7): 8 x FFN(
        (gate_proj): Linear(in_features=256, out_features=512, bias=False)
        (up_proj): Linear(in_features=256, out_features=512, bias=False)
        (down_proj): Linear(in_features=512, out_features=256, bias=False)
        (ffn_dropout): Dropout(p=0.1, inplace=False)
      )
    )

  )
)