Views
No views yet
0.10.0.dev01base_model: Dans-DiscountModels/mistral-7b-v0.3-DanChat
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5trust_remote_code:
6
7# wandb configuration
8wandb_project: 7b-m-dans-personalityengine
9wandb_watch:
10
11wandb_run_id: V1.3.0L-1-8 # V{Version}-{Run Number}-{Attempt Number}
12wandb_log_model:
13
14# push checkpoints to hub
15hub_model_id: Dans-DiscountModels/7b-m-dans-personalityengine-v1.3.0L-TestArticle-1
16# how to push checkpoints to hub
17# https://huggingface.co/docs/transformers/v4.31.0/en/main_classes/trainer#transformers.TrainingArguments.hub_strategy
18hub_strategy: "every_save"
19# Whether to use hf `use_auth_token` for loading datasets. Useful for fetching private datasets
20# Required to be true when used in combination with `push_dataset_to_hub`
21hf_use_auth_token: true
22
23# where to save the finished model to
24output_dir: ./7b-m-dans-personalityengine
25
26# where to save the dataset to
27dataset_prepared_path: ./7b-m-dans-personalityengine-data
28
29save_safetensors: true
30
31# dataset settings (local or huggingface repo)
32datasets:
33 - path: Dans-DiscountModels/dpe-130l-m-7b-32k
34 split: train
35 ds_type: parquet
36 type:
37
38test_datasets:
39 - path: Dans-DiscountModels/dpe-130l-m-7b-32k
40 split: validation
41 ds_type: parquet
42 type:
43
44
45plugins:
46 - axolotl.integrations.liger.LigerPlugin
47 - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
48liger_rope: true
49liger_rms_norm: true
50liger_layer_norm: true
51liger_glu_activation: true
52liger_fused_linear_cross_entropy: false
53cut_cross_entropy: true
54
55load_in_8bit: false
56load_in_4bit: false
57strict: false
58
59sequence_len: 32768
60
61sample_packing: true
62eval_sample_packing: true
63
64pad_to_sequence_len: true
65
66gradient_checkpointing: true
67# gradient_checkpointing_kwargs:
68# use_reentrant: false
69
70gradient_accumulation_steps: 1
71micro_batch_size: 4
72
73num_epochs: 2
74
75optimizer: ademamix_8bit
76optim_args: "beta1=0.9,beta2=0.999,beta3=0.999,alpha=5"
77
78lr_scheduler: rex
79learning_rate: 0.000000012
80cosine_min_lr_ratio: 0.1
81
82# weight_decay: 0.03
83max_grad_norm: 0.001
84
85train_on_inputs: false
86group_by_length: false
87
88bf16: true
89fp16: false
90tf32: false
91
92early_stopping_patience:
93
94resume_from_checkpoint:
95auto_resume_from_checkpoints: false
96
97local_rank:
98logging_steps: 1
99xformers_attention:
100flash_attention: true
101
102warmup_ratio: 0.05
103
104evals_per_epoch: 10
105eval_table_size:
106eval_max_new_tokens:
107
108saves_per_epoch: 2
109save_total_limit: 1
110
111debug: false
112
113deepspeed: deepspeed_configs/zero3_bf16.json
114
115fsdp:
116fsdp_config:
117
118special_tokens:| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.4427 | 0.0021 | 1 | 1.5639 |
| 1.5781 | 0.1015 | 48 | 1.5631 |
| 1.462 | 0.2030 | 96 | 1.5590 |
| 1.6565 | 0.3044 | 144 | 1.5540 |
| 1.454 | 0.4059 | 192 | 1.5498 |
| 1.5414 | 0.5074 | 240 | 1.5471 |
| 1.6084 | 0.6089 | 288 | 1.5459 |
| 1.5315 | 0.7104 | 336 | 1.5457 |
| 1.4646 | 0.8118 | 384 | 1.5465 |
| 1.5506 | 0.9133 | 432 | 1.5482 |
| 1.5083 | 1.0148 | 480 | 1.5506 |
| 1.4986 | 1.1163 | 528 | 1.5538 |
| 1.4976 | 1.2178 | 576 | 1.5576 |
| 1.6139 | 1.3192 | 624 | 1.5618 |
| 1.6305 | 1.4207 | 672 | 1.5666 |
| 1.5522 | 1.5222 | 720 | 1.5717 |
| 1.5846 | 1.6237 | 768 | 1.5771 |
| 1.6093 | 1.7252 | 816 | 1.5824 |
| 1.6282 | 1.8266 | 864 | 1.5873 |
| 1.5984 | 1.9281 | 912 | 1.5911 |