0.4.01base_model: alpindale/Mistral-7B-v0.2-hf
2model_type: MistralForCausalLM
3tokenizer_type: LlamaTokenizer
4is_mistral_derived_model: true
5
6load_in_8bit: false
7load_in_4bit: false
8strict: false
9
10chat_template: chatml
11datasets:
12 - path: data/merged_all.json
13 ds_type: json
14 type: alpaca
15 conversation: chatml
16
17 - path: data/gpteacher-instruct-special-alpaca.json
18 ds_type: json
19 type: gpteacher
20 conversation: chatml
21
22 - path: data/wizardlm_evol_instruct_70k_random_half.json
23 ds_type: json
24 type: alpaca
25 conversation: chatml
26
27 - path: data/capybara_sharegpt.json
28 ds_type: json
29 type: sharegpt
30 conversation: chatml
31
32 - path: data/synthia-v1.3_sharegpt_12500.json
33 ds_type: json
34 type: sharegpt
35 conversation: chatml
36
37 - path: data/cot_alpaca_gpt4_extracted_openhermes_2.5_sharegpt.json
38 ds_type: json
39 type: sharegpt
40 conversation: chatml
41
42 - path: data/slimorca_dedup_filtered_95k_sharegpt.json
43 ds_type: json
44 type: sharegpt
45 conversation: chatml
46
47 - path: data/airoboros_3.2_without_contextual_slimorca_orca_sharegpt.json
48 ds_type: json
49 type: sharegpt
50 conversation: chatml
51
52 - path: data/allenai_wild_chat_gpt4_english_toxic_random_half_4k_sharegpt.json
53 ds_type: json
54 type: sharegpt
55 strict: false
56 conversation: chatml
57
58 - path: data/pippa_bagel_repo_3k_sharegpt.json
59 ds_type: json
60 type: sharegpt
61 conversation: chatml
62
63 - path: data/gpt4_data_lmys_1m_sharegpt.json
64 ds_type: json
65 type: sharegpt
66 conversation: chatml
67
68 - path: data/sharegpt_gpt4_english.json
69 ds_type: json
70 type: sharegpt
71 conversation: chatml
72
73 - path: data/no_robots_sharegpt.json
74 ds_type: json
75 type: sharegpt
76 strict: false
77 conversation: chatml
78
79 - path: data/oasst_top1_from_fusechatmixture_sharegpt.json
80 ds_type: json
81 type: sharegpt
82 strict: false
83 conversation: chatml
84
85 - path: data/everythinglm-data-v3_sharegpt.json
86 ds_type: json
87 type: sharegpt
88 strict: false
89 conversation: chatml
90
91dataset_prepared_path: last_run_prepared
92# val_set_size: 0.005
93val_set_size: 0.0
94
95do_bench_eval: true
96
97output_dir: ./Einstein-v6-7B-model
98
99sequence_len: 8192
100sample_packing: true
101pad_to_sequence_len: true
102eval_sample_packing: false
103
104wandb_project: Einstein
105wandb_entity:
106wandb_watch:
107wandb_name:
108wandb_log_model:
109hub_model_id: Weyaxi/Einstein-v6-7B
110
111save_safetensors: true
112
113gradient_accumulation_steps: 4
114micro_batch_size: 1
115num_epochs: 2
116optimizer: adamw_bnb_8bit
117lr_scheduler: cosine
118learning_rate: 0.000005
119
120train_on_inputs: false
121group_by_length: false
122bf16: true
123fp16: false
124tf32: false
125
126gradient_checkpointing: true
127early_stopping_patience:
128resume_from_checkpoint:
129local_rank:
130logging_steps: 1
131xformers_attention:
132flash_attention: true
133
134warmup_steps: 10
135evals_per_epoch: 3 # changed
136eval_table_size:
137eval_table_max_new_tokens: 128
138saves_per_epoch: 2 # changed
139debug:
140
141deepspeed: zero3_bf16.json
142weight_decay: 0.0
143fsdp:
144fsdp_config:
145special_tokens:
146 bos_token: "<s>"
147 eos_token: "<|im_end|>"
148 unk_token: "<unk>"
149tokens:
150 - "<|im_start|>"<|im_start|>system
{system}<|im_end|>
<|im_start|>user
{user}<|im_end|>
<|im_start|>assistant
{asistant}<|im_end|>tokenizer.apply_chat_template() method:1messages = [
2 {"role": "system", "content": "You are helpful AI asistant."},
3 {"role": "user", "content": "Hello!"}
4]
5gen_input = tokenizer.apply_chat_template(message, return_tensors="pt")
6model.generate(**gen_input)| Metric | Value |
|---|---|
| Avg. | 67.12 |
| AI2 Reasoning Challenge (25-Shot) | 63.57 |
| HellaSwag (10-Shot) | 82.76 |
| MMLU (5-Shot) | 62.23 |
| TruthfulQA (0-shot) | 52.02 |
| Winogrande (5-shot) | 78.61 |
| GSM8k (5-shot) | 63.53 |
