- learning_rate: 1e-5
- lr_scheduler: cosine
- cosine_min_lr_ratio: 0.1
- batch_size(global): 256
- max_seq_length: 32768
- weight_decay: 0.01
- optimizer: paged_adamw_8bit