learning_rate: 5e-06
train_batch_size: 32
eval_batch_size: 64
seed: 211
gradient_accumulation_steps: 8
total_train_batch_size: 256
optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: linear
lr_scheduler_warmup_ratio: 0.05
num_epochs: 20.0

Training Loss	Epoch	Step	Validation Loss
5.3846	0.4225	1000	5.3512
4.8005	0.8450	2000	4.8468
4.3213	1.2675	3000	4.3585
3.9745	1.6900	4000	4.0233
3.6694	2.1124	5000	3.7268
3.4934	2.5349	6000	3.5465
3.3778	2.9574	7000	3.4380
3.2884	3.3798	8000	3.3655
3.2322	3.8024	9000	3.3156
3.1635	4.2248	10000	3.2755
3.1398	4.6473	11000	3.2309
3.0662	5.0697	12000	3.2121
3.0722	5.4922	13000	3.1982
3.0572	5.9148	14000	3.1678
3.0079	6.3372	15000	3.1615
3.0035	6.7597	16000	3.1436
2.9608	7.1821	17000	3.1426
2.9667	7.6046	18000	3.1218
2.927	8.0270	19000	3.1111
2.9288	8.4496	20000	3.1128
2.9298	8.8721	21000	3.1011
2.8912	9.2945	22000	3.1000
2.8984	9.7170	23000	3.0943
2.8563	10.1394	24000	3.0893
2.8709	10.5620	25000	3.0842
2.8709	10.9845	26000	3.0733
2.8391	11.4069	27000	3.0726
2.842	11.8294	28000	3.0672
2.8044	12.2518	29000	3.0726
2.8275	12.6743	30000	3.0623
2.7814	13.0968	31000	3.0642
2.8	13.5193	32000	3.0606
2.7976	13.9418	33000	3.0576
2.7731	14.3642	34000	3.0558
2.7848	14.7867	35000	3.0501
2.7498	15.2091	36000	3.0585
2.7595	15.6317	37000	3.0507
2.7294	16.0541	38000	3.0549
2.7416	16.4766	39000	3.0507
2.7484	16.8991	40000	3.0476
2.7269	17.3215	41000	3.0524
2.7304	17.7441	42000	3.0480
2.7092	18.1665	43000	3.0507
2.7051	18.5890	44000	3.0481
2.7105	19.0114	45000	3.0496
2.7025	19.4339	46000	3.0487
2.7017	19.8564	47000	3.0482

kanishka/opt-babylm1-ntx_seed-211_5e-6

opt-babylm1-ntx_seed-211_5e-6