learning_rate: 1e-05
train_batch_size: 2
eval_batch_size: 2
seed: 42
distributed_type: multi-GPU
num_devices: 3
gradient_accumulation_steps: 16
total_train_batch_size: 96
total_eval_batch_size: 6
optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.98) and epsilon=1e-06 and optimizer_args=No additional optimizer arguments
lr_scheduler_type: polynomial
lr_scheduler_warmup_ratio: 0.1
training_steps: 50000
mixed_precision_training: Native AMP

Training results

Training Loss	Epoch	Step	Contrastive Loss	Diversity Loss	Codevector Perplexity	Validation Loss
125.3126	0.3912	1000	96.4549	40.9977	43.5792	100.5547
118.0391	0.7823	2000	97.6766	41.2832	43.6295	101.8049
115.3469	1.1733	3000	101.3524	41.5331	44.0244	105.5057
112.6943	1.5644	4000	99.6415	41.6922	44.7805	103.8107
109.4814	1.9556	5000	95.8562	41.6997	45.1097	100.0261
109.3386	2.3466	6000	93.9689	40.2005	43.5422	97.9890
107.8054	2.7377	7000	96.9681	41.0117	44.1182	101.0692
106.3722	3.1287	8000	97.0375	41.5678	44.0350	101.1943
104.5718	3.5198	9000	95.1596	41.0475	43.7932	99.2644
103.8211	3.9110	10000	92.1621	40.5181	43.6882	96.2139
103.1223	4.3020	11000	94.1818	41.3664	43.6264	98.3185
104.7649	4.6931	12000	90.9461	41.0200	44.4799	95.0481
103.1965	5.0841	13000	91.9891	41.4620	44.6188	96.1353
101.0915	5.4752	14000	91.8484	41.6080	44.7536	96.0092
102.7036	5.8664	15000	88.1449	41.2676	44.8783	92.2717
101.9447	6.2574	16000	94.4221	41.4548	45.1503	98.5676
100.2287	6.6485	17000	92.4937	40.9730	43.5725	96.5910
100.0868	7.0395	18000	91.7541	40.4903	43.5642	95.8031
101.4023	7.4307	19000	88.1120	41.3487	44.6977	92.2469
99.574	7.8218	20000	93.4205	41.0233	44.3017	97.5228
100.2078	8.2128	21000	91.0207	41.6586	44.5379	95.1866
99.4065	8.6039	22000	88.3160	41.0647	43.7551	92.4225
100.5828	8.9951	23000	89.1667	39.8170	43.4005	93.1484
100.0126	9.3861	24000	89.2179	41.0972	44.1928	93.3276
99.3798	9.7772	25000	90.9742	41.6216	45.3199	95.1364
97.9403	10.1682	26000	90.6132	41.4949	44.9421	94.7627
99.0827	10.5593	27000	90.9845	41.4186	45.5504	95.1264
98.4717	10.9505	28000	85.7275	41.1076	44.3690	89.8383
98.4385	11.3415	29000	89.4880	41.5378	44.4287	93.6418
98.2191	11.7326	30000	89.7903	40.9551	43.5561	93.8858
98.3397	12.1236	31000	88.1321	40.6074	43.7551	92.1928
97.5595	12.5148	32000	88.5040	40.5311	44.1138	92.5571
98.4217	12.9059	33000	88.7710	41.8382	45.2195	92.9548
98.0459	13.2969	34000	89.7865	40.8780	43.8547	93.8743
96.4577	13.6880	35000	86.9712	41.3682	45.3051	91.1080
98.3382	14.0790	36000	86.7875	40.5254	43.3639	90.8401
96.2518	14.4702	37000	89.5054	41.5796	45.0319	93.6634
97.2554	14.8613	38000	87.2171	41.0474	44.8610	91.3218
97.6836	15.2523	39000	89.0243	41.3422	44.1551	93.1585
96.6151	15.6434	40000	87.4906	40.4210	44.1444	91.5327
97.4614	16.0344	41000	92.8991	42.3453	44.9617	97.1336
96.4776	16.4256	42000	89.2628	40.9430	44.3081	93.3571
97.3363	16.8167	43000	87.5446	40.7602	44.4246	91.6206
96.8068	17.2077	44000	88.0595	41.8831	45.3360	92.2479
97.1233	17.5989	45000	86.4747	41.3266	45.5028	90.6074
96.2477	17.9900	46000	87.5056	41.7545	45.2499	91.6811
96.4851	18.3810	47000	87.7251	41.1899	44.1649	91.8441
96.2895	18.7721	48000	90.2903	40.6395	44.7702	94.3543
94.658	19.1631	49000	88.0278	41.2381	44.7100	92.1516
95.3096	19.5543	50000	88.4179	40.7761	44.3206	92.4955

Framework versions

Transformers 4.57.6
Pytorch 2.10.0+cu128
Datasets 2.19.1
Tokenizers 0.22.2

nattkorat/xlsr300m-khmer-cpt-500h

xlsr300m-khmer-cpt-500h

Model description

Intended uses & limitations

Training and evaluation data

Training procedure

Training hyperparameters

Training results

Framework versions

nattkorat/
xlsr300m-khmer-cpt-500h