Views
No views yet
| Name | Quant method | Size |
|---|---|---|
| llama33B.Q2_K.gguf | Q2_K | 1.27GB |
| llama33B.IQ3_XS.gguf | IQ3_XS | 1.38GB |
| llama33B.IQ3_S.gguf | IQ3_S | 1.44GB |
| llama33B.Q3_K_S.gguf | Q3_K_S | 1.44GB |
| llama33B.IQ3_M.gguf | IQ3_M | 1.49GB |
| llama33B.Q3_K.gguf | Q3_K | 1.57GB |
| llama33B.Q3_K_M.gguf | Q3_K_M | 1.57GB |
| llama33B.Q3_K_L.gguf | Q3_K_L | 1.69GB |
| llama33B.IQ4_XS.gguf | IQ4_XS | 1.71GB |
| llama33B.Q4_0.gguf | Q4_0 | 1.79GB |
| llama33B.IQ4_NL.gguf | IQ4_NL | 1.79GB |
| llama33B.Q4_K_S.gguf | Q4_K_S | 1.8GB |
| llama33B.Q4_K.gguf | Q4_K | 1.88GB |
| llama33B.Q4_K_M.gguf | Q4_K_M | 1.88GB |
| llama33B.Q4_1.gguf | Q4_1 | 1.95GB |
| llama33B.Q5_0.gguf | Q5_0 | 2.11GB |
| llama33B.Q5_K_S.gguf | Q5_K_S | 2.11GB |
| llama33B.Q5_K.gguf | Q5_K | 2.16GB |
| llama33B.Q5_K_M.gguf | Q5_K_M | 2.16GB |
| llama33B.Q5_1.gguf | Q5_1 | 2.28GB |
| llama33B.Q6_K.gguf | Q6_K | 2.46GB |
| llama33B.Q8_0.gguf | Q8_0 | 3.19GB |
0.5.01base_model: meta-llama/Llama-3.2-3B
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5plugins:
6 - axolotl.integrations.liger.LigerPlugin
7liger_rope: true
8liger_rms_norm: true
9liger_glu_activation: true
10liger_fused_linear_cross_entropy: true
11
12load_in_8bit: false
13load_in_4bit: false
14strict: false
15
16
17datasets:
18 - path: chrisgru/ro_wiki_chatml_small
19 type: chat_template
20 chat_template: llama3
21 field_messages: conversations
22 message_field_role: from
23 message_field_content: value
24
25dataset_prepared_path: /workspace/data/ds_preprocess
26val_set_size: 0.01
27output_dir: ./data/outputs
28
29sequence_len: 8192
30sample_packing: true
31pad_to_sequence_len: true
32
33
34#adapter: lora
35##lora_model_dir:
36#lora_r: 64
37#lora_alpha: 16
38#lora_dropout: 0.05
39#lora_target_linear: true
40#lora_fan_in_fan_out:
41#lora_modules_to_save:
42# - embed_tokens
43# - lm_head
44
45wandb_project: wiki-llm
46wandb_entity:
47wandb_watch:
48wandb_name:
49wandb_log_model:
50
51gradient_accumulation_steps: 4
52micro_batch_size: 1
53num_epochs: 1
54optimizer: paged_adamw_8bit
55lr_scheduler: cosine
56learning_rate: 5e-5
57
58
59train_on_inputs: false
60group_by_length: false
61bf16: auto
62fp16:
63tf32: false
64
65gradient_checkpointing: true
66gradient_checkpointing_kwargs:
67 use_reentrant: false
68early_stopping_patience:
69resume_from_checkpoint:
70local_rank:
71logging_steps: 1
72xformers_attention:
73flash_attention: true
74
75warmup_steps: 20
76evals_per_epoch: 10
77eval_table_size:
78saves_per_epoch: 1
79#eval_max_new_tokens: 128
80save_total_limit: 2
81debug:
82#deepspeed:
83weight_decay: 0.0
84# fsdp:
85# - full_shard
86# - auto_wrap
87# fsdp_config:
88# fsdp_limit_all_gathers: true
89# fsdp_sync_module_states: true
90# fsdp_offload_params: true
91# fsdp_use_orig_params: false
92# fsdp_cpu_ram_efficient_loading: true
93# fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
94# fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
95# fsdp_state_dict_type: FULL_STATE_DICT
96# fsdp_sharding_strategy: FULL_SHARD
97# fsdp_backward_prefetch: BACKWARD_PRE
98
99seed: 1234
100hf_use_auth_token: true
101hub_strategy: end
102hub_model_id: chrisgru/llama-3.2-3B-rowiki
103special_tokens:
104 bos_token: "<|begin_of_text|>"
105 pad_token: "<|finetune_right_pad_id|>"
106| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.4683 | 0.0009 | 1 | 1.6826 |
| 1.7777 | 0.1001 | 117 | 1.6274 |
| 1.4701 | 0.2003 | 234 | 1.6031 |
| 1.6591 | 0.3004 | 351 | 1.5815 |
| 1.664 | 0.4006 | 468 | 1.5587 |
| 1.5308 | 0.5007 | 585 | 1.5404 |
| 1.3583 | 0.6009 | 702 | 1.5268 |
| 1.4297 | 0.7010 | 819 | 1.5198 |
| 1.7561 | 0.8012 | 936 | 1.5168 |
| 1.6656 | 0.9013 | 1053 | 1.5161 |