Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| completion4B.Q2_K.gguf | Q2_K | 1.71GB |
| completion4B.IQ3_XS.gguf | IQ3_XS | 1.89GB |
| completion4B.IQ3_S.gguf | IQ3_S | 1.97GB |
| completion4B.Q3_K_S.gguf | Q3_K_S | 1.96GB |
| completion4B.IQ3_M.gguf | IQ3_M | 2.03GB |
| completion4B.Q3_K.gguf | Q3_K | 2.14GB |
| completion4B.Q3_K_M.gguf | Q3_K_M | 2.14GB |
| completion4B.Q3_K_L.gguf | Q3_K_L | 2.3GB |
| completion4B.IQ4_XS.gguf | IQ4_XS | 2.38GB |
| completion4B.Q4_0.gguf | Q4_0 | 2.47GB |
| completion4B.IQ4_NL.gguf | IQ4_NL | 2.49GB |
| completion4B.Q4_K_S.gguf | Q4_K_S | 2.48GB |
| completion4B.Q4_K.gguf | Q4_K | 2.59GB |
| completion4B.Q4_K_M.gguf | Q4_K_M | 2.59GB |
| completion4B.Q4_1.gguf | Q4_1 | 2.71GB |
| completion4B.Q5_0.gguf | Q5_0 | 2.95GB |
| completion4B.Q5_K_S.gguf | Q5_K_S | 2.95GB |
| completion4B.Q5_K.gguf | Q5_K | 3.01GB |
| completion4B.Q5_K_M.gguf | Q5_K_M | 3.01GB |
| completion4B.Q5_1.gguf | Q5_1 | 3.19GB |
| completion4B.Q6_K.gguf | Q6_K | 3.46GB |
| completion4B.Q8_0.gguf | Q8_0 | 4.47GB |
0.4.11base_model: IntervitensInc/Llama-3.1-Minitron-4B-Width-Base-chatml
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9hub_model_id: jeiku/completion4B
10hub_strategy: "all_checkpoints"
11push_dataset_to_hub:
12hf_use_auth_token: true
13
14datasets:
15 - path: Mielikki/Erebus-87k
16 type: completion
17 field: body
18
19shuffle_merged_datasets: true
20val_set_size: 0.0025
21output_dir: ./outputs/out
22
23adapter:
24lora_r:
25lora_alpha:
26lora_dropout:
27lora_target_linear:
28
29sequence_len: 8192
30sample_packing: true
31eval_sample_packing: false
32pad_to_sequence_len: true
33
34plugins:
35 - axolotl.integrations.liger.LigerPlugin
36liger_rope: true
37liger_rms_norm: true
38liger_swiglu: true
39liger_fused_linear_cross_entropy: true
40
41wandb_project: EXP4B
42wandb_entity:
43wandb_watch:
44wandb_name: EXP4B
45wandb_log_model:
46
47gradient_accumulation_steps: 12
48micro_batch_size: 3
49num_epochs: 1
50optimizer: adamw_bnb_8bit
51lr_scheduler: cosine
52learning_rate: 0.00001
53weight_decay: 0.05
54
55train_on_inputs: false
56group_by_length: false
57bf16: auto
58fp16:
59tf32: true
60
61gradient_checkpointing: true
62early_stopping_patience:
63resume_from_checkpoint:
64local_rank:
65logging_steps: 1
66xformers_attention:
67flash_attention: true
68
69warmup_ratio: 0.1
70evals_per_epoch: 4
71eval_table_size:
72eval_max_new_tokens: 128
73saves_per_epoch: 1
74
75debug:
76deepspeed: deepspeed_configs/zero3_bf16.json
77fsdp:
78fsdp_config:
79
80special_tokens:
81 pad_token: <|finetune_right_pad_id|>
82| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 2.5227 | 0.0029 | 1 | 2.9798 |
| 2.5027 | 0.2520 | 88 | 2.9501 |
| 2.481 | 0.5039 | 176 | 2.9398 |
| 2.4313 | 0.7559 | 264 | 2.9360 |