learning_rate: 5e-06
train_batch_size: 32
eval_batch_size: 64
seed: 42
gradient_accumulation_steps: 8
total_train_batch_size: 256
optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: linear
lr_scheduler_warmup_ratio: 0.05
num_epochs: 20.0

Training Loss	Epoch	Step	Validation Loss
5.3183	0.4225	1000	5.3184
4.7525	0.8450	2000	4.7876
4.3131	1.2675	3000	4.3554
3.9418	1.6900	4000	3.9805
3.6232	2.1124	5000	3.6927
3.4675	2.5349	6000	3.5203
3.3785	2.9574	7000	3.4430
3.2596	3.3798	8000	3.3456
3.2161	3.8024	9000	3.2887
3.1485	4.2248	10000	3.2598
3.122	4.6473	11000	3.2144
3.0534	5.0697	12000	3.1907
3.0513	5.4922	13000	3.1773
3.046	5.9148	14000	3.1548
3.0028	6.3372	15000	3.1537
2.9961	6.7597	16000	3.1365
2.9485	7.1821	17000	3.1220
2.9494	7.6046	18000	3.1136
2.9189	8.0270	19000	3.1076
2.9224	8.4496	20000	3.0955
2.9084	8.8721	21000	3.0854
2.8743	9.2945	22000	3.0854
2.8859	9.7170	23000	3.0792
2.8406	10.1394	24000	3.0770
2.8587	10.5620	25000	3.0750
2.8606	10.9845	26000	3.0635
2.825	11.4069	27000	3.0683
2.8322	11.8294	28000	3.0524
2.8005	12.2518	29000	3.0592
2.8082	12.6743	30000	3.0493
2.7657	13.0968	31000	3.0556
2.783	13.5193	32000	3.0516
2.7844	13.9418	33000	3.0453
2.7613	14.3642	34000	3.0484
2.7674	14.7867	35000	3.0436
2.7344	15.2091	36000	3.0479
2.7428	15.6317	37000	3.0433
2.7197	16.0541	38000	3.0436
2.7297	16.4766	39000	3.0415
2.7359	16.8991	40000	3.0384
2.7095	17.3215	41000	3.0404
2.7159	17.7441	42000	3.0377
2.7029	18.1665	43000	3.0408
2.7018	18.5890	44000	3.0386
2.6974	19.0114	45000	3.0384
2.692	19.4339	46000	3.0380
2.6932	19.8564	47000	3.0377

kanishka/opt-babylm1-ntx_seed-42_5e-6

opt-babylm1-ntx_seed-42_5e-6