learning_rate: 0.0005
train_batch_size: 32
eval_batch_size: 32
seed: 42
gradient_accumulation_steps: 8
total_train_batch_size: 256
optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: cosine
lr_scheduler_warmup_steps: 1000
num_epochs: 1
mixed_precision_training: Native AMP

Training Loss	Epoch	Step	Validation Loss
2.5553	0.0766	5000	1.7178
1.661	0.1533	10000	1.5099
1.5214	0.2299	15000	1.4135
1.447	0.3065	20000	1.3502
1.3897	0.3832	25000	1.3011
1.3401	0.4598	30000	1.2540
1.2955	0.5365	35000	1.2127
1.25	0.6131	40000	1.1696
1.2098	0.6897	45000	1.1323
1.1726	0.7664	50000	1.1002
1.146	0.8430	55000	1.0771
1.1249	0.9196	60000	1.0652
1.1175	0.9963	65000	1.0623

jin-cheon/codeparrot-ds

codeparrot-ds