1base_model: leveldevai/TurdusBeagle-7B
2model_type: MistralForCausalLM
3tokenizer_type: LlamaTokenizer
4is_mistral_derived_model: true
5
6load_in_8bit: false
7load_in_4bit: false
8strict: false
9
10datasets:
11 - path: shuyuej/metamath_gsm8k
12 type:
13 system_prompt: ""
14 field_instruction: question
15 field_output: answer
16 format: "[INST] {instruction} [/INST]"
17 no_input_format: "[INST] {instruction} [/INST]"
18
19dataset_prepared_path:
20val_set_size: 0.05
21output_dir: ./out
22
23sequence_len: 8192
24sample_packing: true
25pad_to_sequence_len: true
26eval_sample_packing: false
27
28wandb_project:
29wandb_entity:
30wandb_watch:
31wandb_name:
32wandb_log_model:
33
34gradient_accumulation_steps: 4
35micro_batch_size: 2
36num_epochs: 1
37optimizer: adamw_bnb_8bit
38lr_scheduler: cosine
39learning_rate: 0.000005
40
41train_on_inputs: false
42group_by_length: false
43bf16: true
44fp16: false
45tf32: false
46
47gradient_checkpointing: true
48early_stopping_patience:
49resume_from_checkpoint:
50local_rank:
51logging_steps: 1
52xformers_attention:
53flash_attention: true
54
55warmup_steps: 10
56evals_per_epoch: 4
57eval_table_size:
58eval_table_max_new_tokens: 128
59saves_per_epoch: 1
60debug:
61deepspeed:
62weight_decay: 0.0
63fsdp:
64fsdp_config:
65special_tokens:
66 bos_token: "<s>"
67 eos_token: "</s>"
68 unk_token: "<unk>"
69
