Views
No views yet
google/gemma-3-4b-it on the idhant297/faces-vision-alignment dataset.1model = FastVisionModel.get_peft_model(
2 model,
3 finetune_vision_layers = True,
4 finetune_language_layers = True,
5 finetune_attention_modules = True,
6 finetune_mlp_modules = True,
7
8 r = 64,
9 lora_alpha = 32,
10 lora_dropout = 0,
11 bias = "none",
12 random_state = 3407,
13 use_rslora = True,
14 loftq_config = None,
15 target_modules = "all-linear",
16)1from unsloth.trainer import UnslothVisionDataCollator
2import weave
3from trl import SFTTrainer, SFTConfig
4
5FastVisionModel.for_training(model) # Enable for training!
6
7trainer = SFTTrainer(
8 model = model,
9 train_dataset = converted_dataset,
10 processing_class = processor.tokenizer,
11 data_collator = UnslothVisionDataCollator(model, processor),
12 args=SFTConfig(
13 # COMPLETION-ONLY TRAINING (Default behavior for prompt-completion datasets)
14 # completion_only_loss=True is DEFAULT - no need to set explicitly
15
16 # BATCH SIZE - Good as is
17 per_device_train_batch_size = 1,
18 gradient_accumulation_steps = 4, # Effective batch = 8
19
20 # TRAINING DURATION
21 num_train_epochs = 1, # first perform 1 epoch then higher epoch
22
23 # LEARNING RATE
24 learning_rate = 2e-4,
25 max_grad_norm = 0.3, # gradient clipping (0.3–1.0 works; 0.3 is safe)
26
27 # OPTIMIZER
28 optim = "adamw_torch_fused", # stable & memory-efficient (or adamw_torch if you prefer)
29 weight_decay = 0.001, # no decay for LoRA matrices
30
31 # SCHEDULING
32 warmup_steps = 100, # Consider increasing to ~100 for 10k samples
33 # warmup_ratio=0.01, # ~5% of total steps; OR set warmup_steps=int(0.05*total_steps)
34 lr_scheduler_type = "cosine",
35
36 # LOGGING & CHECKPOINTS - Add these!
37 logging_steps = 1,
38 save_strategy = "steps",
39 save_steps = 100,
40 # save_total_limit = 5,
41 load_best_model_at_end = False,
42
43 # DATA EFFICIENCY
44 dataloader_num_workers = 0, # Speed up data loading
45
46 # WANDB
47 report_to = "wandb",
48 run_name = "gemma3-faces-vl-lora-r-64",
49
50 # VISION CONFIG - Keep as is
51 remove_unused_columns = False,
52 dataset_text_field = "",
53 dataset_kwargs = {"skip_prepare_dataset": True},
54 # max_length = max_seq_length,
55 max_length = 2048,
56
57 # RANDOM SEED
58 seed = 3407,
59 output_dir = "outputs-gemma3-faces-1ep-r-64",
60 gradient_checkpointing = True,
61 ),
62)