Views
No views yet
0.4.01base_model: Qwen/Qwen1.5-32B
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: true
7strict: false
8
9chat_template: chatml
10datasets:
11 - path: data/merged_all.json
12 ds_type: json
13 type: alpaca
14 conversation: chatml
15
16 - path: data/capybara_sharegpt.json
17 ds_type: json
18 type: sharegpt
19 conversation: chatml
20
21 - path: data/synthia-v1.3_sharegpt_12500.json
22 ds_type: json
23 type: sharegpt
24 conversation: chatml
25
26 - path: data/cot_alpaca_gpt4_extracted_openhermes_2.5_sharegpt.json
27 ds_type: json
28 type: sharegpt
29 conversation: chatml
30
31 - path: data/slimorca_dedup_filtered_95k_sharegpt.json
32 ds_type: json
33 type: sharegpt
34 conversation: chatml
35
36 - path: data/airoboros_3.2_without_contextual_slimorca_orca_sharegpt.json
37 ds_type: json
38 type: sharegpt
39 conversation: chatml
40
41dataset_prepared_path: last_run_prepared
42val_set_size: 0 # because we won't eval, out of memory :(
43output_dir: ./Einstein-v4-Qwen-1.5-32B-model
44
45sequence_len: 4096
46sample_packing: true
47pad_to_sequence_len: true
48eval_sample_packing: false
49
50adapter: qlora
51lora_model_dir:
52lora_r: 64
53lora_alpha: 32
54lora_dropout: 0.05
55lora_target_linear: true
56lora_fan_in_fan_out:
57lora_modules_to_save:
58 - "embed_tokens"
59 - "lm_head"
60
61wandb_project: Einstein
62wandb_entity:
63wandb_watch:
64wandb_name: Einstein-v4-Qwen-1.5-32B-qlora-2-epoch
65wandb_log_model:
66hub_model_id: Weyaxi/Einstein-v4-Qwen-1.5-32B
67
68save_safetensors: true
69
70gradient_accumulation_steps: 4
71micro_batch_size: 1
72num_epochs: 2
73optimizer: adamw_bnb_8bit
74lr_scheduler: cosine
75learning_rate: 0.0002
76
77train_on_inputs: false
78group_by_length: false
79bf16: true
80fp16: false
81tf32: false
82
83gradient_checkpointing: true
84early_stopping_patience:
85resume_from_checkpoint:
86local_rank:
87logging_steps: 1
88xformers_attention:
89flash_attention: true
90
91warmup_steps: 10
92evals_per_epoch: 0 # because we won't eval, out of memory :(
93eval_table_size:
94eval_table_max_new_tokens: 128
95saves_per_epoch: 2
96debug:
97
98deepspeed: zero3_bf16_cpuoffload_params.json
99weight_decay: 0.0
100fsdp:
101fsdp_config:
102special_tokens:
103 bos_token: "<s>"
104 eos_token: "<|im_end|>"
105 unk_token: "<unk>"
106tokens:
107 - "<|im_start|>"<|im_start|>system
{system}<|im_end|>
<|im_start|>user
{user}<|im_end|>
<|im_start|>assistant
{asistant}<|im_end|>tokenizer.apply_chat_template() method:1messages = [
2 {"role": "system", "content": "You are helpful AI asistant."},
3 {"role": "user", "content": "Hello!"}
4]
5gen_input = tokenizer.apply_chat_template(message, return_tensors="pt")
6model.generate(**gen_input)| Metric | Value |
|---|---|
| Avg. | 68.54 |
| AI2 Reasoning Challenge (25-Shot) | 62.37 |
| HellaSwag (10-Shot) | 83.85 |
| MMLU (5-Shot) | 74.04 |
| TruthfulQA (0-shot) | 58.86 |
| Winogrande (5-shot) | 80.43 |
| GSM8k (5-shot) | 51.71 |
