Views
No views yet
1training_params = SFTConfig(
2 output_dir="checkpoints",
3 per_device_train_batch_size=1,
4 per_device_eval_batch_size=1,
5 gradient_accumulation_steps=2,
6 num_train_epochs=3,
7 learning_rate=1e-4, # lowered from 2e-4 to 1e-4
8 weight_decay=0.001,
9 dataset_text_field="text",
10 report_to="none",
11 bf16=False,
12 fp16=False,
13 dataloader_pin_memory=False,
14 remove_unused_columns=False,
15 max_length=512,
16 gradient_checkpointing=True,
17 dataloader_num_workers=0,
18 save_strategy="epoch",
19 logging_steps=100,
20 average_tokens_across_devices=False # Fix for single device training
21 # Remove loss_type parameter to avoid the warning
22 # The trainer will automatically use ForCausalLMLoss which is correct
23)
24
25# Configure model for gradient checkpointing compatibility
26model.config.use_cache = False
27
28trainer = SFTTrainer(
29 model=model,
30 train_dataset=ds['train'],
31 processing_class=tokenizer,
32 args=training_params
33)1TrainOutput(
2global_step=16773,
3training_loss=2.056998251788356,
4metrics={
5 'train_runtime': 3255.1858,
6 'train_samples_per_second': 10.305,
7 'train_steps_per_second': 5.153,
8 'total_flos': 164188359936000.0,
9 'train_loss': 2.056998251788356})
10