Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| tinyllama_92M.Q2_K.gguf | Q2_K | 0.03GB |
| tinyllama_92M.IQ3_XS.gguf | IQ3_XS | 0.04GB |
| tinyllama_92M.IQ3_S.gguf | IQ3_S | 0.04GB |
| tinyllama_92M.Q3_K_S.gguf | Q3_K_S | 0.04GB |
| tinyllama_92M.IQ3_M.gguf | IQ3_M | 0.04GB |
| tinyllama_92M.Q3_K.gguf | Q3_K | 0.04GB |
| tinyllama_92M.Q3_K_M.gguf | Q3_K_M | 0.04GB |
| tinyllama_92M.Q3_K_L.gguf | Q3_K_L | 0.05GB |
| tinyllama_92M.IQ4_XS.gguf | IQ4_XS | 0.05GB |
| tinyllama_92M.Q4_0.gguf | Q4_0 | 0.05GB |
| tinyllama_92M.IQ4_NL.gguf | IQ4_NL | 0.05GB |
| tinyllama_92M.Q4_K_S.gguf | Q4_K_S | 0.05GB |
| tinyllama_92M.Q4_K.gguf | Q4_K | 0.05GB |
| tinyllama_92M.Q4_K_M.gguf | Q4_K_M | 0.05GB |
| tinyllama_92M.Q4_1.gguf | Q4_1 | 0.05GB |
| tinyllama_92M.Q5_0.gguf | Q5_0 | 0.06GB |
| tinyllama_92M.Q5_K_S.gguf | Q5_K_S | 0.06GB |
| tinyllama_92M.Q5_K.gguf | Q5_K | 0.06GB |
| tinyllama_92M.Q5_K_M.gguf | Q5_K_M | 0.06GB |
| tinyllama_92M.Q5_1.gguf | Q5_1 | 0.06GB |
| tinyllama_92M.Q6_K.gguf | Q6_K | 0.07GB |
| tinyllama_92M.Q8_0.gguf | Q8_0 | 0.09GB |
1 max_seq_len = 256
2 vocab_size = 8192
3 dim = 768
4 n_layers = 12
5 n_heads = 12
6 n_kv_heads = 121 batch_size = 64 # if gradient_accumulation_steps > 1, this is the micro-batch size
2 dropout = 0.0
3 # adamw optimizer
4 gradient_accumulation_steps = 8 # used to simulate larger batch sizes
5 learning_rate = 1e-3 # max learning rate
6 max_iters = 34000 # total number of training iterations
7 weight_decay = 3e-4
8 beta1 = 0.9
9 beta2 = 0.95
10 grad_clip = 1.0 # clip gradients at this value, or disable if == 0.0
11 # learning rate decay settings
12 decay_lr = True # whether to decay the learning rate
13 warmup_iters = 1000 # how many steps to warm up for14xV100 GPUs used.
2Run summary:
3 iter 34000
4 loss/train 0.8704
5 loss/val 0.9966
6 tokens 983040000