Views
No views yet
0.8.01base_model: Dans-DiscountModels/mistral-7b-v0.3-ChatML
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5trust_remote_code:
6
7# wandb configuration
8wandb_project: 7b-m-dans-personalityengine
9wandb_watch:
10
11wandb_run_id: V1.2.1-4-1 # V{Version}-{Run Number}-{Attempt Number}
12wandb_log_model:
13
14# push checkpoints to hub
15hub_model_id: Dans-DiscountModels/7b-m-dans-personalityengine-v1.2.1-rc-5
16# how to push checkpoints to hub
17# https://huggingface.co/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.TrainingArguments.hub_strategy
18hub_strategy: "every_save"
19# Whether to use hf `use_auth_token` for loading datasets. Useful for fetching private datasets
20# Required to be true when used in combination with `push_dataset_to_hub`
21hf_use_auth_token: true
22
23# where to save the finished model to
24output_dir: ./7b-m-dans-personalityengine
25
26# where to save the dataset to
27dataset_prepared_path: ./7b-m-dans-personalityengine-data
28
29save_safetensors: true
30
31# dataset settings (local or huggingface repo)
32datasets:
33 - path: Dans-DiscountModels/pretokenization-test-2
34 ds_type: parquet
35 type:
36
37plugins:
38 - axolotl.integrations.liger.LigerPlugin
39 - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
40liger_rope: true
41liger_rms_norm: true
42liger_layer_norm: true
43liger_glu_activation: true
44liger_fused_linear_cross_entropy: false
45cut_cross_entropy: true
46
47load_in_8bit: false
48load_in_4bit: false
49strict: false
50
51val_set_size: 0.005
52sequence_len: 32768
53
54sample_packing: true
55eval_sample_packing: true
56
57pad_to_sequence_len: true
58
59gradient_checkpointing: true
60# gradient_checkpointing_kwargs:
61# use_reentrant: false
62
63gradient_accumulation_steps: 2
64micro_batch_size: 2
65
66num_epochs: 1
67
68optimizer: ademamix_8bit
69optim_args: "beta1=0.9,beta2=0.999,beta3=0.999,alpha=10"
70
71lr_scheduler: rex
72learning_rate: 0.00000015
73cosine_min_lr_ratio: 0.1
74
75# weight_decay: 0.03
76max_grad_norm: 0.001
77
78train_on_inputs: false
79group_by_length: false
80
81bf16: true
82fp16: false
83tf32: false
84
85early_stopping_patience:
86
87resume_from_checkpoint:
88auto_resume_from_checkpoints: false
89
90local_rank:
91logging_steps: 1
92xformers_attention:
93flash_attention: true
94
95warmup_ratio: 0.03
96
97evals_per_epoch: 24
98eval_table_size:
99eval_max_new_tokens:
100
101saves_per_epoch: 2
102save_total_limit: 1
103
104debug: false
105
106deepspeed: deepspeed_configs/zero3_bf16.json
107
108fsdp:
109fsdp_config:
110
111special_tokens:
112| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.5957 | 0.0007 | 1 | 1.5418 |
| 1.487 | 0.0417 | 61 | 1.4982 |
| 1.5851 | 0.0833 | 122 | 1.4720 |
| 1.3702 | 0.125 | 183 | 1.4596 |
| 1.5285 | 0.1667 | 244 | 1.4519 |
| 1.4809 | 0.2083 | 305 | 1.4461 |
| 1.3806 | 0.25 | 366 | 1.4414 |
| 1.5097 | 0.2917 | 427 | 1.4373 |
| 1.497 | 0.3333 | 488 | 1.4338 |
| 1.503 | 0.375 | 549 | 1.4306 |
| 1.384 | 0.4167 | 610 | 1.4278 |
| 1.4191 | 0.4583 | 671 | 1.4252 |
| 1.3042 | 0.5 | 732 | 1.4228 |
| 1.5669 | 0.5417 | 793 | 1.4206 |
| 1.4239 | 0.5833 | 854 | 1.4185 |
| 1.4472 | 0.625 | 915 | 1.4165 |
| 1.4692 | 0.6667 | 976 | 1.4147 |
| 1.4358 | 0.7083 | 1037 | 1.4130 |
| 1.4676 | 0.75 | 1098 | 1.4114 |
| 1.4657 | 0.7917 | 1159 | 1.4099 |
| 1.424 | 0.8333 | 1220 | 1.4085 |
| 1.3385 | 0.875 | 1281 | 1.4072 |
| 1.4373 | 0.9167 | 1342 | 1.4061 |
| 1.4226 | 0.9583 | 1403 | 1.4052 |
| 1.4225 | 1.0 | 1464 | 1.4047 |