learning_rate: 5e-06
train_batch_size: 32
eval_batch_size: 64
seed: 1024
gradient_accumulation_steps: 8
total_train_batch_size: 256
optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: linear
lr_scheduler_warmup_ratio: 0.05
num_epochs: 20.0

Training Loss	Epoch	Step	Validation Loss
5.2345	0.4205	1000	5.2209
4.6996	0.8410	2000	4.6874
4.2195	1.2616	3000	4.2166
3.8436	1.6821	4000	3.8462
3.5443	2.1026	5000	3.5473
3.3966	2.5231	6000	3.3954
3.3038	2.9437	7000	3.3003
3.2115	3.3642	8000	3.2314
3.1657	3.7847	9000	3.1789
3.0981	4.2052	10000	3.1464
3.0784	4.6257	11000	3.1149
3.0065	5.0463	12000	3.0939
3.0064	5.4668	13000	3.0737
3.0004	5.8873	14000	3.0502
2.9522	6.3078	15000	3.0391
2.9515	6.7283	16000	3.0260
2.9047	7.1489	17000	3.0208
2.9129	7.5694	18000	3.0108
2.913	7.9899	19000	2.9943
2.8739	8.4104	20000	2.9937
2.8815	8.8310	21000	2.9838
2.8349	9.2515	22000	2.9828
2.8455	9.6720	23000	2.9774
2.7954	10.0925	24000	2.9766
2.8167	10.5130	25000	2.9670
2.8199	10.9336	26000	2.9592
2.7855	11.3541	27000	2.9600
2.7894	11.7746	28000	2.9572
2.7552	12.1951	29000	2.9560
2.7671	12.6156	30000	2.9493
2.7306	13.0362	31000	2.9531
2.7471	13.4567	32000	2.9462
2.7515	13.8772	33000	2.9405
2.7206	14.2977	34000	2.9441
2.7313	14.7183	35000	2.9401
2.6916	15.1388	36000	2.9433
2.7093	15.5593	37000	2.9395
2.7164	15.9798	38000	2.9337
2.6972	16.4003	39000	2.9364
2.6923	16.8209	40000	2.9345
2.6756	17.2414	41000	2.9361
2.6778	17.6619	42000	2.9347
2.6609	18.0824	43000	2.9350
2.6592	18.5029	44000	2.9335
2.6664	18.9235	45000	2.9317
2.6549	19.3440	46000	2.9330
2.6535	19.7645	47000	2.9327

kanishka/opt-babylm1_more_entrenchment_laugh_seed-1024_5e-6

opt-babylm1_more_entrenchment_laugh_seed-1024_5e-6