1base_model: mistralai/Mistral-7B-v0.1
2# optionally might have model_type or tokenizer_type
3model_type: MistralForCausalLM
4tokenizer_type: LlamaTokenizer
5# Automatically upload checkpoint and final model to HF
6hub_model_id: AiAF/UFOs-Pretraining-V1.1
7
8load_in_8bit: false
9load_in_4bit: false
10strict: false
11
12datasets:
13 - path: AiAF/pretraining.jsonl
14 type: completion
15
16dataset_prepared_path:
17val_set_size: 0.05
18output_dir: ./outputs/out/v1.1
19
20sequence_len: 8192
21sample_packing: true
22pad_to_sequence_len: true
23eval_sample_packing: false
24
25max_steps: 100000
26
27wandb_project: "UFO_LLM_Pretraining"
28wandb_entity:
29wandb_watch: "all"
30wandb_name: "UFO_LLM_Pretraining-V1.1"
31wandb_log_model: "false"
32
33gradient_accumulation_steps: 4
34micro_batch_size: 2
35num_epochs: 10
36optimizer: adamw_bnb_8bit
37lr_scheduler: cosine
38learning_rate: 0.000005
39
40train_on_inputs: false
41group_by_length: false
42bf16: auto
43fp16:
44tf32: false
45
46gradient_checkpointing: true
47early_stopping_patience:
48resume_from_checkpoint:
49local_rank:
50logging_steps: 1
51xformers_attention:
52flash_attention: true
53
54warmup_steps: 10
55evals_per_epoch: 4
56eval_table_size:
57eval_max_new_tokens: 128
58saves_per_epoch: 1
59debug:
60deepspeed:
61weight_decay: 0.0
62fsdp:
63fsdp_config:
64special_tokens:
65
66
