0.4.01base_model: microsoft/phi-2
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9chat_template: chatml
10datasets:
11 - path: data/merged_all.json
12 ds_type: json
13 type: alpaca
14 conversation: chatml
15
16 - path: data/capybara_sharegpt.json
17 ds_type: json
18 type: sharegpt
19 conversation: chatml
20
21 - path: data/synthia-v1.3_sharegpt_12500.json
22 ds_type: json
23 type: sharegpt
24 conversation: chatml
25
26 - path: data/cot_alpaca_gpt4_extracted_openhermes_2.5_sharegpt.json
27 ds_type: json
28 type: sharegpt
29 conversation: chatml
30
31 - path: data/slimorca_dedup_filtered_95k_sharegpt.json
32 ds_type: json
33 type: sharegpt
34 conversation: chatml
35
36 - path: data/airoboros_3.2_without_contextual_slimorca_orca_sharegpt.json
37 ds_type: json
38 type: sharegpt
39 conversation: chatml
40
41dataset_prepared_path: last_run_prepared
42val_set_size: 0.005
43output_dir: ./Einstein-v4-phi2-model
44
45sequence_len: 2048
46sample_packing: true
47pad_to_sequence_len: true
48eval_sample_packing: false
49
50wandb_project: Einstein
51wandb_entity:
52wandb_watch:
53wandb_name:
54wandb_log_model:
55hub_model_id: Weyaxi/Einstein-v4-phi2
56
57save_safetensors: true
58
59gradient_accumulation_steps: 4
60micro_batch_size: 3
61num_epochs: 2
62optimizer: adamw_torch # adamw_bnb_8bit
63lr_scheduler: cosine
64learning_rate: 0.000005
65
66train_on_inputs: false
67group_by_length: false
68bf16: true
69fp16: false
70tf32: false
71
72gradient_checkpointing: true
73early_stopping_patience:
74resume_from_checkpoint:
75local_rank:
76logging_steps: 1
77xformers_attention:
78flash_attention: true
79
80warmup_steps: 10
81evals_per_epoch: 2 # changed
82eval_table_size:
83eval_table_max_new_tokens: 128
84saves_per_epoch: 4
85debug:
86
87deepspeed: zero3_bf16.json
88weight_decay: 0.0
89fsdp:
90fsdp_config:
91special_tokens:
92 eos_token: "<|im_end|>"
93 pad_token: "<|endoftext|>"
94tokens:
95 - "<|im_start|>"<|im_start|>system
{system}<|im_end|>
<|im_start|>user
{user}<|im_end|>
<|im_start|>assistant
{asistant}<|im_end|>tokenizer.apply_chat_template() method:1messages = [
2 {"role": "system", "content": "You are helpful AI asistant."},
3 {"role": "user", "content": "Hello!"}
4]
5gen_input = tokenizer.apply_chat_template(message, return_tensors="pt")
6model.generate(**gen_input)| Metric | Value |
|---|---|
| Avg. | 60.77 |
| AI2 Reasoning Challenge (25-Shot) | 59.98 |
| HellaSwag (10-Shot) | 74.07 |
| MMLU (5-Shot) | 56.89 |
| TruthfulQA (0-shot) | 45.80 |
| Winogrande (5-shot) | 73.88 |
| GSM8k (5-shot) | 53.98 |
