Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| instructered4B.Q2_K.gguf | Q2_K | 1.71GB |
| instructered4B.IQ3_XS.gguf | IQ3_XS | 1.89GB |
| instructered4B.IQ3_S.gguf | IQ3_S | 1.97GB |
| instructered4B.Q3_K_S.gguf | Q3_K_S | 1.96GB |
| instructered4B.IQ3_M.gguf | IQ3_M | 2.03GB |
| instructered4B.Q3_K.gguf | Q3_K | 2.14GB |
| instructered4B.Q3_K_M.gguf | Q3_K_M | 2.14GB |
| instructered4B.Q3_K_L.gguf | Q3_K_L | 2.3GB |
| instructered4B.IQ4_XS.gguf | IQ4_XS | 2.38GB |
| instructered4B.Q4_0.gguf | Q4_0 | 2.47GB |
| instructered4B.IQ4_NL.gguf | IQ4_NL | 2.49GB |
| instructered4B.Q4_K_S.gguf | Q4_K_S | 2.48GB |
| instructered4B.Q4_K.gguf | Q4_K | 2.59GB |
| instructered4B.Q4_K_M.gguf | Q4_K_M | 2.59GB |
| instructered4B.Q4_1.gguf | Q4_1 | 2.71GB |
| instructered4B.Q5_0.gguf | Q5_0 | 2.95GB |
| instructered4B.Q5_K_S.gguf | Q5_K_S | 2.95GB |
| instructered4B.Q5_K.gguf | Q5_K | 3.01GB |
| instructered4B.Q5_K_M.gguf | Q5_K_M | 3.01GB |
| instructered4B.Q5_1.gguf | Q5_1 | 3.19GB |
| instructered4B.Q6_K.gguf | Q6_K | 3.46GB |
| instructered4B.Q8_0.gguf | Q8_0 | 4.47GB |
0.4.11base_model: jeiku/completion4B
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9hub_model_id: jeiku/instructered4B
10hub_strategy: "all_checkpoints"
11push_dataset_to_hub:
12hf_use_auth_token: true
13
14datasets:
15 - path: FourOhFour/Instruct_Phase
16 type: sharegpt
17 conversation: chatml
18
19chat_template: chatml
20
21shuffle_merged_datasets: true
22val_set_size: 0.0025
23output_dir: ./outputs/out
24
25adapter:
26lora_r:
27lora_alpha:
28lora_dropout:
29lora_target_linear:
30
31sequence_len: 8192
32sample_packing: true
33eval_sample_packing: false
34pad_to_sequence_len: true
35
36plugins:
37 - axolotl.integrations.liger.LigerPlugin
38liger_rope: true
39liger_rms_norm: true
40liger_swiglu: true
41liger_fused_linear_cross_entropy: true
42
43wandb_project: EXP4B
44wandb_entity:
45wandb_watch:
46wandb_name: EXP4B
47wandb_log_model:
48
49gradient_accumulation_steps: 12
50micro_batch_size: 3
51num_epochs: 2
52optimizer: adamw_bnb_8bit
53lr_scheduler: cosine
54learning_rate: 0.00001
55weight_decay: 0.05
56
57train_on_inputs: false
58group_by_length: false
59bf16: auto
60fp16:
61tf32: true
62
63gradient_checkpointing: true
64early_stopping_patience:
65resume_from_checkpoint:
66local_rank:
67logging_steps: 1
68xformers_attention:
69flash_attention: true
70
71warmup_ratio: 0.1
72evals_per_epoch: 4
73eval_table_size:
74eval_max_new_tokens: 128
75saves_per_epoch: 2
76
77debug:
78deepspeed: deepspeed_configs/zero3_bf16.json
79fsdp:
80fsdp_config:
81
82special_tokens:
83 pad_token: <|finetune_right_pad_id|>
84| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.336 | 0.0029 | 1 | 1.7114 |
| 0.9631 | 0.2516 | 86 | 1.4098 |
| 0.9347 | 0.5032 | 172 | 1.3828 |
| 0.9142 | 0.7548 | 258 | 1.3693 |
| 0.7967 | 1.0037 | 344 | 1.3659 |
| 0.7912 | 1.2551 | 430 | 1.3728 |
| 0.7957 | 1.5065 | 516 | 1.3730 |
| 0.7951 | 1.7579 | 602 | 1.3713 |