learning_rate: 5e-05
train_batch_size: 16
eval_batch_size: 16
seed: 42
optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: linear
lr_scheduler_warmup_steps: 198
training_steps: 19800
mixed_precision_training: Native AMP

Training Loss	Epoch	Step	Validation Loss	Accuracy
6.1154	0.1033	200	5.0417	0.2987
5.2001	0.2066	400	4.6315	0.3037
4.9324	0.3099	600	4.4495	0.3151
4.6366	0.4132	800	4.3532	0.3213
4.3878	0.5165	1000	4.3022	0.3213
4.3467	0.6198	1200	4.2962	0.3158
4.1575	0.7231	1400	4.2556	0.3160
4.1211	0.8264	1600	4.1590	0.3356
4.0085	0.9298	1800	4.1952	0.3251
3.8398	1.0331	2000	4.1387	0.3313
2.5393	2.0661	4000	3.9741	0.3627
2.1302	3.0992	6000	3.8979	0.3778
1.8989	4.1322	8000	3.7969	0.3872
1.7936	5.1653	10000	3.8104	0.3886
1.6629	6.1983	12000	3.8619	0.3863
1.5851	7.2314	14000	3.8960	0.3865
1.4878	8.2645	16000	3.9092	0.3867
1.4398	9.2975	18000	3.9346	0.3855

alexandertam/babylm-base1m-gpt2

babylm-base1m-gpt2